diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 19c8ad6e..344478c2 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -74,7 +74,7 @@ def make_environment(config): os.write(write_file_descriptor, passphrase.encode('utf-8')) os.close(write_file_descriptor) - # This, plus subprocess.Popen(..., close_fds=False) in execute.py, is necessary for the Borg + # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the Borg # child process to inherit the file descriptor. os.set_inheritable(read_file_descriptor, True) environment['BORG_PASSPHRASE_FD'] = str(read_file_descriptor) diff --git a/borgmatic/execute.py b/borgmatic/execute.py index df58c833..20eedb58 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -266,8 +266,8 @@ def log_command(full_command, input_file=None, output_file=None, environment=Non width=MAX_LOGGED_COMMAND_LENGTH, placeholder=' ...', ) - + (f" < {getattr(input_file, 'name', '')}" if input_file else '') - + (f" > {getattr(output_file, 'name', '')}" if output_file else '') + + (f" < {getattr(input_file, 'name', input_file)}" if input_file else '') + + (f" > {getattr(output_file, 'name', output_file)}" if output_file else '') ) @@ -315,8 +315,8 @@ def execute_command( shell=shell, env=environment, cwd=working_directory, - # Necessary for the passcommand credential hook to work. - close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')), + # Necessary for passing credentials via anonymous pipe. + close_fds=False, ) if not run_to_completion: return process @@ -333,6 +333,7 @@ def execute_command( def execute_command_and_capture_output( full_command, + input_file=None, capture_stderr=False, shell=False, environment=None, @@ -342,28 +343,30 @@ def execute_command_and_capture_output( ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its - output (stdout). If capture stderr is True, then capture and return stderr in addition to - stdout. If shell is True, execute the command within a shell. If an environment variables dict - is given, then pass it into the command. If a working directory is given, use that as the - present working directory when running the command. If a Borg local path is given, and the - command matches it (regardless of arguments), treat exit code 1 as a warning instead of an - error. But if Borg exit codes are given as a sequence of exit code configuration dicts, then use - that configuration to decide what's an error and what's a warning. + output (stdout). If an input file descriptor is given, then pipe it to the command's stdin. If + capture stderr is True, then capture and return stderr in addition to stdout. If shell is True, + execute the command within a shell. If an environment variables dict is given, then pass it into + the command. If a working directory is given, use that as the present working directory when + running the command. If a Borg local path is given, and the command matches it (regardless of + arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are given + as a sequence of exit code configuration dicts, then use that configuration to decide what's an + error and what's a warning. Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' - log_command(full_command, environment=environment) + log_command(full_command, input_file, environment=environment) command = ' '.join(full_command) if shell else full_command try: output = subprocess.check_output( command, + stdin=input_file, stderr=subprocess.STDOUT if capture_stderr else None, shell=shell, env=environment, cwd=working_directory, - # Necessary for the passcommand credential hook to work. - close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')), + # Necessary for passing credentials via anonymous pipe. + close_fds=False, ) except subprocess.CalledProcessError as error: if ( @@ -422,8 +425,8 @@ def execute_command_with_processes( shell=shell, env=environment, cwd=working_directory, - # Necessary for the passcommand credential hook to work. - close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')), + # Necessary for passing credentials via anonymous pipe. + close_fds=False, ) except (subprocess.CalledProcessError, OSError): # Something has gone wrong. So vent each process' output buffer to prevent it from hanging. diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index b992a944..fc879142 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -1,6 +1,7 @@ import copy import logging import os +import re import shlex import borgmatic.borg.pattern @@ -23,14 +24,90 @@ def make_dump_path(base_directory): # pragma: no cover return dump.make_data_source_dump_path(base_directory, 'mariadb_databases') -SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') +DEFAULTS_EXTRA_FILE_FLAG_PATTERN = re.compile('^--defaults-extra-file=(?P.*)$') -def database_names_to_dump(database, config, environment, dry_run): +def parse_extra_options(extra_options): ''' - Given a requested database config and a configuration dict, return the corresponding sequence of - database names to dump. In the case of "all", query for the names of databases on the configured - host and return them, excluding any system databases that will cause problems during restore. + Given an extra options string, split the options into a tuple and return it. Additionally, if + the first option is "--defaults-extra-file=...", then remove it from the options and return the + filename. + + So the return value is a tuple of: (parsed options, defaults extra filename). + + The intent is to support downstream merging of multiple "--defaults-extra-file"s, as + MariaDB/MySQL only allows one at a time. + ''' + split_extra_options = tuple(shlex.split(extra_options)) if extra_options else () + + if not split_extra_options: + return ((), None) + + match = DEFAULTS_EXTRA_FILE_FLAG_PATTERN.match(split_extra_options[0]) + + if not match: + return (split_extra_options, None) + + return (split_extra_options[1:], match.group('filename')) + + +def make_defaults_file_options(username=None, password=None, defaults_extra_filename=None): + ''' + Given a database username and/or password, write it to an anonymous pipe and return the flags + for passing that file descriptor to an executed command. The idea is that this is a more secure + way to transmit credentials to a database client than using an environment variable. + + If no username or password are given, then return the options for the given defaults extra + filename (if any). But if there is a username and/or password and a defaults extra filename is + given, then "!include" it from the generated file, effectively allowing multiple defaults extra + files. + + Do not use the returned value for multiple different command invocations. That will not work + because each pipe is "used up" once read. + ''' + values = '\n'.join( + ( + (f'user={username}' if username is not None else ''), + (f'password={password}' if password is not None else ''), + ) + ).strip() + + if not values: + if defaults_extra_filename: + return (f'--defaults-extra-file={defaults_extra_filename}',) + + return () + + fields_message = ' and '.join( + field_name + for field_name in ( + (f'username ({username})' if username is not None else None), + ('password' if password is not None else None), + ) + if field_name is not None + ) + include_message = f' (including {defaults_extra_filename})' if defaults_extra_filename else '' + logger.debug(f'Writing database {fields_message} to defaults extra file pipe{include_message}') + + include = f'!include {defaults_extra_filename}\n' if defaults_extra_filename else '' + + read_file_descriptor, write_file_descriptor = os.pipe() + os.write(write_file_descriptor, f'{include}[client]\n{values}'.encode('utf-8')) + os.close(write_file_descriptor) + + # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database + # client child process to inherit the file descriptor. + os.set_inheritable(read_file_descriptor, True) + + return (f'--defaults-extra-file=/dev/fd/{read_file_descriptor}',) + + +def database_names_to_dump(database, config, username, password, environment, dry_run): + ''' + Given a requested database config, a configuration dict, a database username and password, an + environment dict, and whether this is a dry run, return the corresponding sequence of database + names to dump. In the case of "all", query for the names of databases on the configured host and + return them, excluding any system databases that will cause problems during restore. ''' if database['name'] != 'all': return (database['name'],) @@ -40,24 +117,20 @@ def database_names_to_dump(database, config, environment, dry_run): mariadb_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb') ) + extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) show_command = ( mariadb_show_command - + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + ( - ( - '--user', - borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), - ) - if 'username' in database - else () - ) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) + logger.debug('Querying for "all" MariaDB databases to dump') + show_output = execute_command_and_capture_output(show_command, environment=environment) return tuple( @@ -67,8 +140,19 @@ def database_names_to_dump(database, config, environment, dry_run): ) +SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') + + def execute_dump_command( - database, config, dump_path, database_names, environment, dry_run, dry_run_label + database, + config, + username, + password, + dump_path, + database_names, + environment, + dry_run, + dry_run_label, ): ''' Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named @@ -95,21 +179,15 @@ def execute_dump_command( shlex.quote(part) for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump') ) + extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) dump_command = ( mariadb_dump_command - + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + ( - ( - '--user', - borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), - ) - if 'username' in database - else () - ) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -165,19 +243,16 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - environment = dict( - os.environ, - **( - { - 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( - database['password'], config - ) - } - if 'password' in database - else {} - ), + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) + environment = dict(os.environ) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run ) - dump_database_names = database_names_to_dump(database, config, environment, dry_run) if not dump_database_names: if dry_run: @@ -193,6 +268,8 @@ def dump_data_sources( execute_dump_command( renamed_database, config, + username, + password, dump_path, (dump_name,), environment, @@ -205,6 +282,8 @@ def dump_data_sources( execute_dump_command( database, config, + username, + password, dump_path, dump_database_names, environment, @@ -296,20 +375,17 @@ def restore_data_source_dump( mariadb_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb') ) + extra_options, defaults_extra_filename = parse_extra_options(data_source.get('restore_options')) restore_command = ( mariadb_restore_command + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + ('--batch',) - + ( - tuple(data_source['restore_options'].split(' ')) - if 'restore_options' in data_source - else () - ) + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) - + (('--user', username) if username else ()) ) - environment = dict(os.environ, **({'MYSQL_PWD': password} if password else {})) + environment = dict(os.environ) logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}") if dry_run: diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 3172edbb..13d77041 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -6,6 +6,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.hooks.credential.parse +import borgmatic.hooks.data_source.mariadb from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -26,11 +27,12 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, config, environment, dry_run): +def database_names_to_dump(database, config, username, password, environment, dry_run): ''' - Given a requested database config and a configuration dict, return the corresponding sequence of - database names to dump. In the case of "all", query for the names of databases on the configured - host and return them, excluding any system databases that will cause problems during restore. + Given a requested database config, a configuration dict, a database username and password, an + environment dict, and whether this is a dry run, return the corresponding sequence of database + names to dump. In the case of "all", query for the names of databases on the configured host and + return them, excluding any system databases that will cause problems during restore. ''' if database['name'] != 'all': return (database['name'],) @@ -40,24 +42,24 @@ def database_names_to_dump(database, config, environment, dry_run): mysql_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql') ) + extra_options, defaults_extra_filename = ( + borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options')) + ) show_command = ( mysql_show_command - + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename + ) + + extra_options + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + ( - ( - '--user', - borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), - ) - if 'username' in database - else () - ) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) + logger.debug('Querying for "all" MySQL databases to dump') + show_output = execute_command_and_capture_output(show_command, environment=environment) return tuple( @@ -68,7 +70,15 @@ def database_names_to_dump(database, config, environment, dry_run): def execute_dump_command( - database, config, dump_path, database_names, environment, dry_run, dry_run_label + database, + config, + username, + password, + dump_path, + database_names, + environment, + dry_run, + dry_run_label, ): ''' Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a @@ -94,21 +104,19 @@ def execute_dump_command( mysql_dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump') ) + extra_options, defaults_extra_filename = ( + borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options')) + ) dump_command = ( mysql_dump_command - + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename + ) + + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + ( - ( - '--user', - borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), - ) - if 'username' in database - else () - ) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -164,19 +172,16 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - environment = dict( - os.environ, - **( - { - 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( - database['password'], config - ) - } - if 'password' in database - else {} - ), + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) + environment = dict(os.environ) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run ) - dump_database_names = database_names_to_dump(database, config, environment, dry_run) if not dump_database_names: if dry_run: @@ -192,6 +197,8 @@ def dump_data_sources( execute_dump_command( renamed_database, config, + username, + password, dump_path, (dump_name,), environment, @@ -204,6 +211,8 @@ def dump_data_sources( execute_dump_command( database, config, + username, + password, dump_path, dump_database_names, environment, @@ -295,20 +304,21 @@ def restore_data_source_dump( mysql_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql') ) + extra_options, defaults_extra_filename = ( + borgmatic.hooks.data_source.mariadb.parse_extra_options(data_source.get('restore_options')) + ) restore_command = ( mysql_restore_command - + ('--batch',) - + ( - tuple(data_source['restore_options'].split(' ')) - if 'restore_options' in data_source - else () + + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename ) + + extra_options + + ('--batch',) + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) - + (('--user', username) if username else ()) ) - environment = dict(os.environ, **({'MYSQL_PWD': password} if password else {})) + environment = dict(os.environ) logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}") if dry_run: diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 769d4dbf..30a3590f 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -309,10 +309,8 @@ hooks: ### External passwords If you don't want to keep your database passwords in your borgmatic -configuration file, you can instead pass them in via [environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/) -or command-line [configuration -overrides](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides). +configuration file, you can instead pass them in [from external credential +sources](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). ### Configuration backups diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 474c7d6d..fe183865 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -6,10 +6,108 @@ from flexmock import flexmock from borgmatic.hooks.data_source import mariadb as module +def test_parse_extra_options_passes_through_empty_options(): + assert module.parse_extra_options('') == ((), None) + + +def test_parse_extra_options_with_defaults_extra_file_removes_and_and_parses_out_filename(): + assert module.parse_extra_options('--defaults-extra-file=extra.cnf --skip-ssl') == ( + ('--skip-ssl',), + 'extra.cnf', + ) + + +def test_parse_extra_options_without_defaults_extra_file_passes_through_options(): + assert module.parse_extra_options('--skip-ssl --and=stuff') == ( + ('--skip-ssl', '--and=stuff'), + None, + ) + + +def test_make_defaults_file_pipe_without_username_or_password_bails(): + flexmock(module.os).should_receive('pipe').never() + + assert module.make_defaults_file_options(username=None, password=None) == () + + +def test_make_defaults_file_option_with_username_and_password_writes_them_to_file_descriptor(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\nuser=root\npassword=trustsome1' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options(username='root', password='trustsome1') == ( + '--defaults-extra-file=/dev/fd/99', + ) + + +def test_make_defaults_file_pipe_with_only_username_writes_it_to_file_descriptor(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\nuser=root' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options(username='root', password=None) == ( + '--defaults-extra-file=/dev/fd/99', + ) + + +def test_make_defaults_file_pipe_with_only_password_writes_it_to_file_descriptor(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\npassword=trustsome1' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options(username=None, password='trustsome1') == ( + '--defaults-extra-file=/dev/fd/99', + ) + + +def test_make_defaults_file_option_with_defaults_extra_filename_includes_it_in_file_descriptor(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'!include extra.cnf\n[client]\nuser=root\npassword=trustsome1' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options( + username='root', password='trustsome1', defaults_extra_filename='extra.cnf' + ) == ('--defaults-extra-file=/dev/fd/99',) + + +def test_make_defaults_file_option_with_only_defaults_extra_filename_uses_it_instead_of_file_descriptor(): + flexmock(module.os).should_receive('pipe').never() + + assert module.make_defaults_file_options( + username=None, password=None, defaults_extra_filename='extra.cnf' + ) == ('--defaults-extra-file=extra.cnf',) + + def test_database_names_to_dump_passes_through_name(): environment = flexmock() - names = module.database_names_to_dump({'name': 'foo'}, {}, environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'foo'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo',) @@ -18,7 +116,9 @@ def test_database_names_to_dump_bails_for_dry_run(): environment = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=True) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=True + ) assert names == () @@ -28,12 +128,25 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'), + ( + 'mariadb', + '--defaults-extra-file=/dev/fd/99', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), environment=environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo', 'bar') @@ -53,6 +166,9 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' @@ -65,6 +181,8 @@ def test_dump_data_sources_dumps_each_database(): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, config={}, + username=None, + password=None, dump_path=object, database_names=(name,), environment={'USER': 'root'}, @@ -89,10 +207,10 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -100,9 +218,11 @@ def test_dump_data_sources_dumps_with_password(): flexmock(module).should_receive('execute_dump_command').with_args( database=database, config={}, + username='root', + password='trustsome1', dump_path=object, database_names=('foo',), - environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -121,14 +241,16 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, config={}, + username=None, + password=None, dump_path=object, database_names=('foo', 'bar'), environment={'USER': 'root'}, @@ -150,16 +272,18 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value, config: value) + ).and_return(None) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, config={}, + username=None, + password=None, dump_path=object, database_names=(name,), environment={'USER': 'root'}, @@ -181,11 +305,18 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured def test_database_names_to_dump_runs_mariadb_with_list_options(): - database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf'} + database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl'} + flexmock(module).should_receive('parse_extra_options').and_return( + ('--skip-ssl',), 'mariadb.cnf' + ) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', 'mariadb.cnf' + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mariadb', - '--defaults-extra-file=mariadb.cnf', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -194,20 +325,30 @@ def test_database_names_to_dump_runs_mariadb_with_list_options(): environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): database = { 'name': 'all', - 'list_options': '--defaults-extra-file=mariadb.cnf', + 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl', 'mariadb_command': 'custom_mariadb', } + flexmock(module).should_receive('parse_extra_options').and_return( + ('--skip-ssl',), 'mariadb.cnf' + ) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', 'mariadb.cnf' + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( environment=None, full_command=( 'custom_mariadb', # Custom MariaDB command - '--defaults-extra-file=mariadb.cnf', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -215,7 +356,10 @@ def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_execute_dump_command_runs_mariadb_dump(): @@ -225,11 +369,16 @@ def test_execute_dump_command_runs_mariadb_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--databases', 'foo', @@ -244,6 +393,8 @@ def test_execute_dump_command_runs_mariadb_dump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -261,11 +412,16 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--databases', 'foo', '--result-file', @@ -279,6 +435,8 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -296,11 +454,16 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--host', 'database.example.org', @@ -321,6 +484,8 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -338,20 +503,23 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', - '--user', - 'root', '--databases', 'foo', '--result-file', 'dump', ), - environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, run_to_completion=False, ).and_return(process).once() @@ -359,9 +527,11 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, dry_run=False, dry_run_label='', ) @@ -376,11 +546,16 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return(('--stuff=such',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--stuff=such', '--add-drop-database', '--databases', @@ -396,6 +571,8 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -413,11 +590,16 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return(('--stuff=such',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'custom_mariadb_dump', # Custom MariaDB dump command + '--defaults-extra-file=/dev/fd/99', '--stuff=such', '--add-drop-database', '--databases', @@ -437,6 +619,8 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): 'options': '--stuff=such', }, # Custom MariaDB dump command specified config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -450,6 +634,10 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -457,6 +645,8 @@ def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -473,6 +663,10 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -481,6 +675,8 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -546,6 +742,10 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch'), @@ -578,9 +778,13 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return(('--harder',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mariadb', '--batch', '--harder'), + ('mariadb', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -612,9 +816,13 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return(('--harder',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('custom_mariadb', '--batch', '--harder'), + ('custom_mariadb', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -644,6 +852,10 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -685,13 +897,17 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mariadb', '--batch', '--user', 'root'), + ('mariadb', '--defaults-extra-file=/dev/fd/99', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -727,10 +943,15 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'cliusername', 'clipassword', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'clihost', @@ -738,13 +959,11 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ 'cliport', '--protocol', 'tcp', - '--user', - 'cliusername', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'clipassword'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -782,10 +1001,15 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'restoreuser', 'restorepass', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'restorehost', @@ -793,13 +1017,11 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', - '--user', - 'restoreuser', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'restorepass'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -824,6 +1046,10 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 5b4cbcc9..2e629dc3 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -9,7 +9,9 @@ from borgmatic.hooks.data_source import mysql as module def test_database_names_to_dump_passes_through_name(): environment = flexmock() - names = module.database_names_to_dump({'name': 'foo'}, {}, environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'foo'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo',) @@ -21,7 +23,9 @@ def test_database_names_to_dump_bails_for_dry_run(): ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=True) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=True + ) assert names == () @@ -31,12 +35,27 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'), + ( + 'mysql', + '--defaults-extra-file=/dev/fd/99', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), environment=environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo', 'bar') @@ -56,6 +75,9 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) @@ -65,6 +87,8 @@ def test_dump_data_sources_dumps_each_database(): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, config={}, + username=None, + password=None, dump_path=object, database_names=(name,), environment={'USER': 'root'}, @@ -89,10 +113,10 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -100,9 +124,11 @@ def test_dump_data_sources_dumps_with_password(): flexmock(module).should_receive('execute_dump_command').with_args( database=database, config={}, + username='root', + password='trustsome1', dump_path=object, database_names=('foo',), - environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -121,11 +147,16 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, config={}, + username=None, + password=None, dump_path=object, database_names=('foo', 'bar'), environment={'USER': 'root'}, @@ -147,6 +178,9 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) @@ -154,6 +188,8 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, config={}, + username=None, + password=None, dump_path=object, database_names=(name,), environment={'USER': 'root'}, @@ -175,11 +211,18 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured def test_database_names_to_dump_runs_mysql_with_list_options(): - database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf'} + database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf --skip-ssl'} + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--skip-ssl',), 'my.cnf') + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', 'my.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mysql', - '--defaults-extra-file=my.cnf', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -188,20 +231,30 @@ def test_database_names_to_dump_runs_mysql_with_list_options(): environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): database = { 'name': 'all', - 'list_options': '--defaults-extra-file=my.cnf', + 'list_options': '--defaults-extra-file=my.cnf --skip-ssl', 'mysql_command': 'custom_mysql', } + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--skip-ssl',), 'my.cnf') + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', 'my.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( environment=None, full_command=( 'custom_mysql', # Custom MySQL command - '--defaults-extra-file=my.cnf', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -209,7 +262,10 @@ def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_execute_dump_command_runs_mysqldump(): @@ -219,11 +275,18 @@ def test_execute_dump_command_runs_mysqldump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--databases', 'foo', @@ -238,6 +301,8 @@ def test_execute_dump_command_runs_mysqldump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -255,11 +320,18 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--databases', 'foo', '--result-file', @@ -273,6 +345,8 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -290,11 +364,18 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--host', 'database.example.org', @@ -315,6 +396,8 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -332,20 +415,25 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', - '--user', - 'root', '--databases', 'foo', '--result-file', 'dump', ), - environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, run_to_completion=False, ).and_return(process).once() @@ -353,9 +441,11 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, dry_run=False, dry_run_label='', ) @@ -370,11 +460,18 @@ def test_execute_dump_command_runs_mysqldump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--stuff=such',), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--stuff=such', '--add-drop-database', '--databases', @@ -390,6 +487,8 @@ def test_execute_dump_command_runs_mysqldump_with_options(): module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -407,11 +506,18 @@ def test_execute_dump_command_runs_non_default_mysqldump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'custom_mysqldump', # Custom MySQL dump command + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--databases', 'foo', @@ -429,6 +535,8 @@ def test_execute_dump_command_runs_non_default_mysqldump(): 'mysql_dump_command': 'custom_mysqldump', }, # Custom MySQL dump command specified config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -442,6 +550,12 @@ def test_execute_dump_command_runs_non_default_mysqldump(): def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -449,6 +563,8 @@ def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -465,6 +581,12 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -473,6 +595,8 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -538,6 +662,12 @@ def test_restore_data_source_dump_runs_mysql_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch'), @@ -570,9 +700,15 @@ def test_restore_data_source_dump_runs_mysql_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--harder',), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mysql', '--batch', '--harder'), + ('mysql', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -602,9 +738,15 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--harder',), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('custom_mysql', '--batch', '--harder'), + ('custom_mysql', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -634,6 +776,12 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -675,13 +823,19 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mysql', '--batch', '--user', 'root'), + ('mysql', '--defaults-extra-file=/dev/fd/99', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -717,10 +871,19 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('cliusername', 'clipassword', None).and_return( + ('--defaults-extra-file=/dev/fd/99',) + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'clihost', @@ -728,13 +891,11 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ 'cliport', '--protocol', 'tcp', - '--user', - 'cliusername', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'clipassword'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -772,10 +933,19 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('restoreuser', 'restorepass', None).and_return( + ('--defaults-extra-file=/dev/fd/99',) + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'restorehost', @@ -783,13 +953,11 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', - '--user', - 'restoreuser', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'restorepass'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -814,6 +982,12 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 4d39fe3b..f3ef9413 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -191,7 +191,7 @@ def test_execute_command_calls_full_command(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -213,7 +213,7 @@ def test_execute_command_calls_full_command_with_output_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stderr=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -234,7 +234,7 @@ def test_execute_command_calls_full_command_without_capturing_output(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(wait=lambda: 0)).once() flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS) flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) @@ -257,7 +257,7 @@ def test_execute_command_calls_full_command_with_input_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -278,7 +278,7 @@ def test_execute_command_calls_full_command_with_shell(): shell=True, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -299,7 +299,7 @@ def test_execute_command_calls_full_command_with_environment(): shell=False, env={'a': 'b'}, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -320,33 +320,12 @@ def test_execute_command_calls_full_command_with_working_directory(): shell=False, env=None, cwd='/working', - close_fds=True, - ).and_return(flexmock(stdout=None)).once() - flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('log_outputs') - - output = module.execute_command(full_command, working_directory='/working') - - assert output is None - - -def test_execute_command_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): - full_command = ['foo', 'bar'] - flexmock(module).should_receive('log_command') - flexmock(module.subprocess).should_receive('Popen').with_args( - full_command, - stdin=None, - stdout=module.subprocess.PIPE, - stderr=module.subprocess.STDOUT, - shell=False, - env={'BORG_PASSPHRASE_FD': '4'}, - cwd=None, close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') - output = module.execute_command(full_command, environment={'BORG_PASSPHRASE_FD': '4'}) + output = module.execute_command(full_command, working_directory='/working') assert output is None @@ -363,7 +342,7 @@ def test_execute_command_without_run_to_completion_returns_process(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(process).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -377,11 +356,12 @@ def test_execute_command_and_capture_output_returns_stdout(): flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output(full_command) @@ -395,11 +375,12 @@ def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr() flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=module.subprocess.STDOUT, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output(full_command, capture_stderr=True) @@ -414,11 +395,12 @@ def test_execute_command_and_capture_output_returns_output_when_process_error_is flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_raise(subprocess.CalledProcessError(1, full_command, err_output)).once() flexmock(module).should_receive('interpret_exit_code').and_return( module.Exit_status.SUCCESS @@ -435,11 +417,12 @@ def test_execute_command_and_capture_output_raises_when_command_errors(): flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_raise(subprocess.CalledProcessError(2, full_command, expected_output)).once() flexmock(module).should_receive('interpret_exit_code').and_return( module.Exit_status.ERROR @@ -455,11 +438,12 @@ def test_execute_command_and_capture_output_returns_output_with_shell(): flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( 'foo bar', + stdin=None, stderr=None, shell=True, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output(full_command, shell=True) @@ -473,11 +457,12 @@ def test_execute_command_and_capture_output_returns_output_with_environment(): flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env={'a': 'b'}, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( @@ -493,37 +478,16 @@ def test_execute_command_and_capture_output_returns_output_with_working_director flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd='/working', - close_fds=True, - ).and_return(flexmock(decode=lambda: expected_output)).once() - - output = module.execute_command_and_capture_output( - full_command, shell=False, working_directory='/working' - ) - - assert output == expected_output - - -def test_execute_command_and_capture_output_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): - full_command = ['foo', 'bar'] - expected_output = '[]' - flexmock(module).should_receive('log_command') - flexmock(module.subprocess).should_receive('check_output').with_args( - full_command, - stderr=None, - shell=False, - env={'BORG_PASSPHRASE_FD': '4'}, - cwd=None, close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( - full_command, - shell=False, - environment={'BORG_PASSPHRASE_FD': '4'}, + full_command, shell=False, working_directory='/working' ) assert output == expected_output @@ -541,7 +505,7 @@ def test_execute_command_with_processes_calls_full_command(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -564,7 +528,7 @@ def test_execute_command_with_processes_returns_output_with_output_log_level_non shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(process).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs').and_return({process: 'out'}) @@ -587,7 +551,7 @@ def test_execute_command_with_processes_calls_full_command_with_output_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stderr=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -609,7 +573,7 @@ def test_execute_command_with_processes_calls_full_command_without_capturing_out shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(wait=lambda: 0)).once() flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS) flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) @@ -635,7 +599,7 @@ def test_execute_command_with_processes_calls_full_command_with_input_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -657,7 +621,7 @@ def test_execute_command_with_processes_calls_full_command_with_shell(): shell=True, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -679,7 +643,7 @@ def test_execute_command_with_processes_calls_full_command_with_environment(): shell=False, env={'a': 'b'}, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -701,39 +665,13 @@ def test_execute_command_with_processes_calls_full_command_with_working_director shell=False, env=None, cwd='/working', - close_fds=True, - ).and_return(flexmock(stdout=None)).once() - flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('log_outputs') - - output = module.execute_command_with_processes( - full_command, processes, working_directory='/working' - ) - - assert output is None - - -def test_execute_command_with_processes_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): - full_command = ['foo', 'bar'] - processes = (flexmock(),) - flexmock(module).should_receive('log_command') - flexmock(module.subprocess).should_receive('Popen').with_args( - full_command, - stdin=None, - stdout=module.subprocess.PIPE, - stderr=module.subprocess.STDOUT, - shell=False, - env={'BORG_PASSPHRASE_FD': '4'}, - cwd=None, close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes( - full_command, - processes, - environment={'BORG_PASSPHRASE_FD': '4'}, + full_command, processes, working_directory='/working' ) assert output is None @@ -754,7 +692,7 @@ def test_execute_command_with_processes_kills_processes_on_error(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_raise(subprocess.CalledProcessError(1, full_command, 'error')).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs').never()