Merge branch 'main' into parallel-borgmatic

NEWS

@@ -1,4 +1,9 @@
 2.1.6.dev0
+ * #1267: Support multiple borgmatic instances run in parallel—as long as they're operating on
+   different Borg repositories. This required an internal change to the runtime directory path,
+   which may cause a one-time performance hit (Borg cache misses) for Borg 1.x users of database and
+   filesystem hooks. See the documentation for more information:
+   https://torsion.org/borgmatic/how-to/make-per-application-backups/#parallelism
  * #1300: Fix the "source_directories_must_exist" option to support source directories relative to a
    "working_directory".
  * #1301: Expand the "patterns_from" and "exclude_from" options to support paths containing tildes

borgmatic/actions/check.py

@@ -788,7 +788,7 @@ def run_check(
     '''
     logger.info('Running consistency checks')
 
-    repository_id = borgmatic.borg.check.get_repository_id(
+    repository_id = borgmatic.borg.repo_info.get_repository_id(
         repository['path'],
         config,
         local_borg_version,
@@ -844,7 +844,9 @@ def run_check(
 
     if 'spot' in checks:
         logger.info('Running spot check')
-        with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+        with borgmatic.config.paths.Runtime_directory(
+            config, repository_id
+        ) as borgmatic_runtime_directory:
             spot_check(
                 repository,
                 config,

borgmatic/actions/config/bootstrap.py

@@ -3,6 +3,7 @@ import logging
 import os
 
 import borgmatic.borg.extract
+import borgmatic.borg.repo_info
 import borgmatic.borg.repo_list
 import borgmatic.config.paths
 
@@ -109,8 +110,18 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
         local_path=bootstrap_arguments.local_path,
         remote_path=bootstrap_arguments.remote_path,
     )
+    repository_id = borgmatic.borg.repo_info.get_repository_id(
+        bootstrap_arguments.repository,
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=bootstrap_arguments.local_path,
+        remote_path=bootstrap_arguments.remote_path,
+    )
 
-    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+    with borgmatic.config.paths.Runtime_directory(
+        config, repository_id
+    ) as borgmatic_runtime_directory:
         manifest_config_paths = load_config_paths_from_archive(
             bootstrap_arguments.repository,
             archive_name,

borgmatic/actions/create.py

@@ -42,8 +42,18 @@ def run_create(
 
     logger.info(f'Creating archive{dry_run_label}')
     working_directory = borgmatic.config.paths.get_working_directory(config)
+    repository_id = borgmatic.borg.repo_info.get_repository_id(
+        repository['path'],
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )
 
-    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+    with borgmatic.config.paths.Runtime_directory(
+        config, repository_id
+    ) as borgmatic_runtime_directory:
         patterns = pattern.process_patterns(
             pattern.collect_patterns(config, working_directory),
             config,

borgmatic/actions/restore.py

@@ -536,8 +536,18 @@ def run_restore(
     '''
     logger.info(f'Restoring data sources from archive {restore_arguments.archive}')
     working_directory = borgmatic.config.paths.get_working_directory(config)
+    repository_id = borgmatic.borg.repo_info.get_repository_id(
+        repository['path'],
+        config,
+        local_borg_version,
+        global_arguments,
+        local_path=local_path,
+        remote_path=remote_path,
+    )
 
-    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+    with borgmatic.config.paths.Runtime_directory(
+        config, repository_id
+    ) as borgmatic_runtime_directory:
         patterns = borgmatic.actions.pattern.process_patterns(
             borgmatic.actions.pattern.collect_patterns(config, working_directory),
             config,

borgmatic/borg/check.py

@@ -1,10 +1,8 @@
-import argparse
-import json
 import logging
 import shlex
 
 import borgmatic.config.paths
-from borgmatic.borg import environment, feature, flags, repo_info
+from borgmatic.borg import environment, feature, flags
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
@@ -82,36 +80,6 @@ def make_check_name_flags(checks, archive_filter_flags):
     )
 
 
-def get_repository_id(
-    repository_path,
-    config,
-    local_borg_version,
-    global_arguments,
-    local_path,
-    remote_path,
-):
-    '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, global
-    arguments, and local/remote commands to run, return the corresponding Borg repository ID.
-
-    Raise ValueError if the Borg repository ID cannot be determined.
-    '''
-    try:
-        return json.loads(
-            repo_info.display_repository_info(
-                repository_path,
-                config,
-                local_borg_version,
-                argparse.Namespace(json=True),
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-        )['repository']['id']
-    except (json.JSONDecodeError, KeyError):
-        raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
-
-
 def check_archives(
     repository_path,
     config,

borgmatic/borg/repo_info.py

@@ -1,3 +1,5 @@
+import argparse
+import json
 import logging
 import shlex
 
@@ -81,3 +83,33 @@ def display_repository_info(
     )
 
     return None
+
+
+def get_repository_id(
+    repository_path,
+    config,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Given a local or remote repository path, a configuration dict, the local Borg version, global
+    arguments, and local/remote commands to run, return the corresponding Borg repository ID.
+
+    Raise ValueError if the Borg repository ID cannot be determined.
+    '''
+    try:
+        return json.loads(
+            display_repository_info(
+                repository_path,
+                config,
+                local_borg_version,
+                argparse.Namespace(json=True),
+                global_arguments,
+                local_path,
+                remote_path,
+            ),
+        )['repository']['id']
+    except (json.JSONDecodeError, KeyError):
+        raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')

borgmatic/config/paths.py

@@ -1,6 +1,7 @@
 import contextlib
 import logging
 import os
+import shutil
 import tempfile
 from enum import Enum
 
@@ -84,6 +85,25 @@ def replace_temporary_subdirectory_with_glob(
     )
 
 
+class Fixed_name_temporary_directory:
+    '''
+    A class whose instances can stand-in for tempfile.TemporaryDirectory's, except the temporary
+    filename path is fixed rather than randomly generated.
+    '''
+
+    def __init__(self, path):
+        '''
+        Given a temporary directory path, save it off for later.
+        '''
+        self.name = path
+
+    def cleanup(self):
+        '''
+        Remove the temporary directory path.
+        '''
+        shutil.rmtree(self.name, ignore_errors=True)
+
+
 class Runtime_directory:
     '''
     A Python context manager for creating and cleaning up the borgmatic runtime directory used for
@@ -98,16 +118,17 @@ class Runtime_directory:
     automatically gets cleaned up as necessary.
     '''
 
-    def __init__(self, config):
+    def __init__(self, config, repository_id):
         '''
-        Given a configuration dict determine the borgmatic runtime directory, creating a secure,
+        Given a configuration dict and the Borg ID for a repository, determine the borgmatic runtime
-        temporary directory within it if necessary. Defaults to $XDG_RUNTIME_DIR/./borgmatic or
+        directory, creating a secure, temporary directory within it if necessary. Defaults to
-        $RUNTIME_DIRECTORY/./borgmatic or $TMPDIR/borgmatic-[random]/./borgmatic or
+        $XDG_RUNTIME_DIR/borgmatic-[repository_id]/./borgmatic or
-        $TEMP/borgmatic-[random]/./borgmatic or /tmp/borgmatic-[random]/./borgmatic where "[random]"
+        $RUNTIME_DIRECTORY/borgmatic-[repository_id]/./borgmatic or
-        is a randomly generated string intended to avoid path collisions.
+        $TMPDIR/borgmatic-[random]/./borgmatic or $TEMP/borgmatic-[random]/./borgmatic or
-
+        /tmp/borgmatic-[random]/./borgmatic where "[random]" is a randomly generated string and
-        If XDG_RUNTIME_DIR or RUNTIME_DIRECTORY is set and already ends in "/borgmatic", then don't
+        "[repository_id]" is the Borg repository ID being operated on. Both strings are intended to
-        tack on a second "/borgmatic" path component.
+        avoid path collisions, and the random string helps avoid predictable temporary path attacks
+        in shared temporary directories.
 
         The "/./" is taking advantage of a Borg feature such that the part of the path before the "/./"
         does not get stored in the file path within an archive. That way, the path of the runtime
@@ -125,7 +146,8 @@ class Runtime_directory:
             if not runtime_directory.startswith(os.path.sep):
                 raise ValueError('The runtime directory must be an absolute path')
 
-            self.temporary_directory = None
+            runtime_directory = os.path.join(runtime_directory, f'borgmatic-{repository_id}')
+            self.temporary_directory = Fixed_name_temporary_directory(runtime_directory)
         else:
             base_directory = (
                 os.environ.get('TMPDIR') or os.environ.get('TEMP') or '/tmp'  # noqa: S108
@@ -141,11 +163,9 @@ class Runtime_directory:
             )
             runtime_directory = self.temporary_directory.name
 
-        (base_path, final_directory) = os.path.split(runtime_directory.rstrip(os.path.sep))
-
         self.runtime_path = expand_user_in_path(
             os.path.join(
-                base_path if final_directory == 'borgmatic' else runtime_directory,
+                runtime_directory,
                 '.',  # Borg 1.4+ "slashdot" hack.
                 'borgmatic',
             ),
@@ -162,14 +182,13 @@ class Runtime_directory:
 
     def __exit__(self, exception_type, exception, traceback):
         '''
-        Delete any temporary directory that was created as part of initialization.
+        Delete the temporary directory that was created as part of initialization.
         '''
-        if self.temporary_directory:
+        # The cleanup() call errors if, for instance, there's still a
-            # The cleanup() call errors if, for instance, there's still a
+        # mounted filesystem within the temporary directory. There's
-            # mounted filesystem within the temporary directory. There's
+        # nothing we can do about that here, so swallow the error.
-            # nothing we can do about that here, so swallow the error.
+        with contextlib.suppress(OSError):
-            with contextlib.suppress(OSError):
+            self.temporary_directory.cleanup()
-                self.temporary_directory.cleanup()
 
 
 def make_runtime_directory_glob(borgmatic_runtime_directory):

docs/how-to/make-per-application-backups.md

@@ -58,19 +58,25 @@ choice](https://torsion.org/borgmatic/how-to/set-up-backups/#autopilot), each
 entry using borgmatic's `--config` flag instead of relying on
 `/etc/borgmatic.d`.
 
+<a id="limitations"></a>
 
-## Limitations
+## Parallelism
 
-borgmatic does not currently support its own parallelism—being run multiple
+<span class="minilink minilink-addedin">New in version 2.1.6</span> borgmatic
-times on the same machine simultaneously. In particular, many of the [data
+supports multiple borgmatic instances run on the same machine in parallel—as
-source
+long as they're operating on different Borg repositories.
-hooks](https://torsion.org/borgmatic/reference/configuration/data-sources/) rely
-on global borgmatic runtime files which can't be shared across processes, and
-therefore multiple borgmatic instances on the same machine would interfere with
-each other.
 
-A single borgmatic instance also doesn't currently support running multiple Borg
+However, note that a single borgmatic instance doesn't currently support running
-instances in parallel on the same machine.
+multiple Borg instances in parallel on the same machine.
+
+
+<span class="minilink minilink-addedin">Prior to version 2.1.6</span> borgmatic
+did not support parallel borgmatic runs on the same machine simultaneously. In
+particular, many of the [data source
+hooks](https://torsion.org/borgmatic/reference/configuration/data-sources/)
+relied on global borgmatic runtime files which couldn't be shared across
+processes, and therefore multiple borgmatic instances on the same machine
+interfered with each other.
 
 
 <a id="archive-naming"></a>

docs/reference/configuration/runtime-directory.md

@@ -29,11 +29,18 @@ probes the following values:
 You can see the runtime directory path that borgmatic selects by running with
 `--verbosity 2` and looking for `Using runtime directory` in the output.
 
-Regardless of the runtime directory selected, borgmatic stores its files
+Regardless of the runtime directory selected, borgmatic stores its files within
-within a `borgmatic` subdirectory of the runtime directory. Additionally, in
+a `borgmatic` subdirectory of the runtime directory. Additionally, in the case
-the case of `TMPDIR`, `TEMP`, and the hard-coded `/tmp`, borgmatic creates a
+of `TMPDIR`, `TEMP`, and the hard-coded `/tmp`, borgmatic creates a randomly
-randomly named subdirectory in an effort to reduce path collisions in shared
+named subdirectory in an effort to reduce path collisions and predictable
-system temporary directories.
+temporary path attacks in shared temporary directories.
+
+<span class="minilink minilink-addedin">New in version 2.1.6</span> When
+constructing the runtime directory, borgmatic now creates a subdirectory named
+after the Borg ID of the current repository. This means that borgmatic supports
+multiple borgmatic instances run [in
+parallel](https://torsion.org/borgmatic/how-to/make-per-application-backups/#parallelism)—as
+long as they're operating on different Borg repositories.
 
 <span class="minilink minilink-addedin">Prior to version 1.9.0</span>
 borgmatic created temporary streaming database dumps within the `~/.borgmatic`

tests/unit/actions/config/test_bootstrap.py

@@ -171,6 +171,7 @@ def test_run_bootstrap_does_not_raise():
     flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
         'archive',
     )
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('repo')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         flexmock(),
     )
@@ -222,6 +223,7 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config():
         local_path='borg7',
         remote_path='borg8',
     ).and_return('archive')
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('repo')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         flexmock(),
     )

tests/unit/actions/test_check.py

@@ -1838,7 +1838,7 @@ def test_spot_check_without_any_source_paths_errors():
 
 def test_run_check_checks_archives_for_configured_repository():
     flexmock(module.logger).answer = lambda message: None
-    flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module).should_receive('upgrade_check_times')
     flexmock(module).should_receive('parse_checks')
     flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
@@ -1873,7 +1873,7 @@ def test_run_check_checks_archives_for_configured_repository():
 
 def test_run_check_runs_configured_extract_check():
     flexmock(module.logger).answer = lambda message: None
-    flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module).should_receive('upgrade_check_times')
     flexmock(module).should_receive('parse_checks')
     flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
@@ -1906,7 +1906,7 @@ def test_run_check_runs_configured_extract_check():
 
 def test_run_check_runs_configured_spot_check():
     flexmock(module.logger).answer = lambda message: None
-    flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module).should_receive('upgrade_check_times')
     flexmock(module).should_receive('parse_checks')
     flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
@@ -1942,7 +1942,7 @@ def test_run_check_runs_configured_spot_check():
 
 def test_run_check_without_checks_runs_nothing_except_hooks():
     flexmock(module.logger).answer = lambda message: None
-    flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module).should_receive('upgrade_check_times')
     flexmock(module).should_receive('parse_checks')
     flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())

tests/unit/actions/test_create.py

@@ -11,6 +11,7 @@ from borgmatic.actions import create as module
 
 def test_run_create_executes_and_calls_hooks_for_configured_repository():
     flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         flexmock(),
     )
@@ -49,6 +50,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
 
 def test_run_create_with_both_list_and_json_errors():
     flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').never()
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never()
     flexmock(module.borgmatic.borg.create).should_receive('create_archive').never()
     create_arguments = flexmock(
@@ -80,6 +82,7 @@ def test_run_create_with_both_list_and_json_errors():
 
 def test_run_create_with_both_list_and_progress_errors():
     flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').never()
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never()
     flexmock(module.borgmatic.borg.create).should_receive('create_archive').never()
     create_arguments = flexmock(
@@ -111,6 +114,7 @@ def test_run_create_with_both_list_and_progress_errors():
 
 def test_run_create_produces_json():
     flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         flexmock(),
     )
@@ -156,6 +160,7 @@ def test_run_create_with_active_dumps_roundtrips_via_checkpoint_archive():
     mock_dump_process.should_receive('poll').and_return(None).and_return(0)
 
     flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         flexmock(),
     )
@@ -234,6 +239,7 @@ def test_run_create_with_active_dumps_json_updates_archive_info():
     }
 
     flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         flexmock(),
     )
@@ -316,6 +322,7 @@ def mock_dump_cleanup(config, borgmatic_runtime_directory, patterns, dry_run):
 
 def test_run_create_with_active_dumps_removes_data_source_dumps_with_original_patterns():
     flexmock(module.logger).answer = lambda message: None
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         flexmock(),
     )

tests/unit/actions/test_restore.py

@@ -1175,6 +1175,7 @@ def test_run_restore_restores_each_data_source():
     }
 
     borgmatic_runtime_directory = flexmock()
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         borgmatic_runtime_directory,
     )
@@ -1249,6 +1250,7 @@ def test_run_restore_restores_data_source_by_falling_back_to_all_name():
     }
 
     borgmatic_runtime_directory = flexmock()
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         borgmatic_runtime_directory,
     )
@@ -1311,6 +1313,7 @@ def test_run_restore_restores_data_source_configured_with_all_name():
     }
 
     borgmatic_runtime_directory = flexmock()
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         borgmatic_runtime_directory,
     )
@@ -1395,6 +1398,7 @@ def test_run_restore_skips_missing_data_source():
     }
 
     borgmatic_runtime_directory = flexmock()
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         borgmatic_runtime_directory,
     )
@@ -1479,6 +1483,7 @@ def test_run_restore_restores_data_sources_from_different_hooks():
     }
 
     borgmatic_runtime_directory = flexmock()
+    flexmock(module.borgmatic.borg.repo_info).should_receive('get_repository_id').and_return('id')
     flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
         borgmatic_runtime_directory,
     )

tests/unit/borg/test_check.py

@@ -272,54 +272,6 @@ def test_make_check_name_flags_with_extract_omits_extract_flag():
     assert flags == ()
 
 
-def test_get_repository_id_with_valid_json_does_not_raise():
-    config = {}
-    flexmock(module.repo_info).should_receive('display_repository_info').and_return(
-        '{"repository": {"id": "repo"}}',
-    )
-
-    assert module.get_repository_id(
-        repository_path='repo',
-        config=config,
-        local_borg_version='1.2.3',
-        global_arguments=flexmock(),
-        local_path='borg',
-        remote_path=None,
-    )
-
-
-def test_get_repository_id_with_json_error_raises():
-    config = {}
-    flexmock(module.repo_info).should_receive('display_repository_info').and_return(
-        '{"unexpected": {"id": "repo"}}',
-    )
-
-    with pytest.raises(ValueError):
-        module.get_repository_id(
-            repository_path='repo',
-            config=config,
-            local_borg_version='1.2.3',
-            global_arguments=flexmock(),
-            local_path='borg',
-            remote_path=None,
-        )
-
-
-def test_get_repository_id_with_missing_json_keys_raises():
-    config = {}
-    flexmock(module.repo_info).should_receive('display_repository_info').and_return('{invalid JSON')
-
-    with pytest.raises(ValueError):
-        module.get_repository_id(
-            repository_path='repo',
-            config=config,
-            local_borg_version='1.2.3',
-            global_arguments=flexmock(),
-            local_path='borg',
-            remote_path=None,
-        )
-
-
 def test_check_archives_with_progress_passes_through_to_borg():
     config = {'progress': True}
     flexmock(module).should_receive('make_check_name_flags').with_args(

tests/unit/borg/test_repo_info.py

@@ -1,5 +1,6 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import repo_info as module
@@ -612,3 +613,51 @@ def test_display_repository_info_calls_borg_with_working_directory():
         repo_info_arguments=flexmock(json=False),
         global_arguments=flexmock(),
     )
+
+
+def test_get_repository_id_with_valid_json_does_not_raise():
+    config = {}
+    flexmock(module).should_receive('display_repository_info').and_return(
+        '{"repository": {"id": "repo"}}',
+    )
+
+    assert module.get_repository_id(
+        repository_path='repo',
+        config=config,
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(),
+        local_path='borg',
+        remote_path=None,
+    )
+
+
+def test_get_repository_id_with_json_error_raises():
+    config = {}
+    flexmock(module).should_receive('display_repository_info').and_return(
+        '{"unexpected": {"id": "repo"}}',
+    )
+
+    with pytest.raises(ValueError):
+        module.get_repository_id(
+            repository_path='repo',
+            config=config,
+            local_borg_version='1.2.3',
+            global_arguments=flexmock(),
+            local_path='borg',
+            remote_path=None,
+        )
+
+
+def test_get_repository_id_with_missing_json_keys_raises():
+    config = {}
+    flexmock(module).should_receive('display_repository_info').and_return('{invalid JSON')
+
+    with pytest.raises(ValueError):
+        module.get_repository_id(
+            repository_path='repo',
+            config=config,
+            local_borg_version='1.2.3',
+            global_arguments=flexmock(),
+            local_path='borg',
+            remote_path=None,
+        )

tests/unit/config/test_paths.py

@@ -62,65 +62,58 @@ def test_replace_temporary_subdirectory_with_glob_uses_custom_temporary_director
     )
 
 
+def test_fixed_name_temporary_directory_cleanup_does_not_raise():
+    flexmock(module.shutil).should_receive('rmtree')
+
+    module.Fixed_name_temporary_directory('/path').cleanup()
+
+
 def test_runtime_directory_uses_config_option():
+    flexmock(module).should_receive('Fixed_name_temporary_directory').and_return(
+        flexmock(cleanup=lambda: None)
+    )
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
     flexmock(module.os).should_receive('makedirs')
     config = {'user_runtime_directory': '/run', 'borgmatic_source_directory': '/nope'}
 
-    with module.Runtime_directory(config) as borgmatic_runtime_directory:
+    with module.Runtime_directory(config, repository_id='id') as borgmatic_runtime_directory:
-        assert borgmatic_runtime_directory == '/run/./borgmatic'
+        assert borgmatic_runtime_directory == '/run/borgmatic-id/./borgmatic'
-
-
-def test_runtime_directory_uses_config_option_without_adding_duplicate_borgmatic_subdirectory():
-    flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os).should_receive('makedirs')
-    config = {'user_runtime_directory': '/run/borgmatic', 'borgmatic_source_directory': '/nope'}
-
-    with module.Runtime_directory(config) as borgmatic_runtime_directory:
-        assert borgmatic_runtime_directory == '/run/./borgmatic'
 
 
 def test_runtime_directory_with_relative_config_option_errors():
     flexmock(module.os).should_receive('makedirs').never()
     config = {'user_runtime_directory': 'run', 'borgmatic_source_directory': '/nope'}
 
-    with pytest.raises(ValueError), module.Runtime_directory(config):
+    with pytest.raises(ValueError), module.Runtime_directory(config, repository_id='id'):
         pass
 
 
-def test_runtime_directory_falls_back_to_xdg_runtime_dir():
+def test_runtime_directory_falls_back_to_xdg_runtime_dir(monkeypatch):
-    flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
+    flexmock(module).should_receive('Fixed_name_temporary_directory').and_return(
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(
+        flexmock(cleanup=lambda: None)
-        '/run',
     )
+    flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
+    monkeypatch.setenv('XDG_RUNTIME_DIR', '/run')
     flexmock(module.os).should_receive('makedirs')
 
-    with module.Runtime_directory({}) as borgmatic_runtime_directory:
+    with module.Runtime_directory({}, repository_id='id') as borgmatic_runtime_directory:
-        assert borgmatic_runtime_directory == '/run/./borgmatic'
+        assert borgmatic_runtime_directory == '/run/borgmatic-id/./borgmatic'
 
 
-def test_runtime_directory_falls_back_to_xdg_runtime_dir_without_adding_duplicate_borgmatic_subdirectory():
+def test_runtime_directory_with_relative_xdg_runtime_dir_errors(monkeypatch):
-    flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
+    monkeypatch.setenv('XDG_RUNTIME_DIR', 'run')
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(
-        '/run/borgmatic',
-    )
-    flexmock(module.os).should_receive('makedirs')
-
-    with module.Runtime_directory({}) as borgmatic_runtime_directory:
-        assert borgmatic_runtime_directory == '/run/./borgmatic'
-
-
-def test_runtime_directory_with_relative_xdg_runtime_dir_errors():
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return('run')
     flexmock(module.os).should_receive('makedirs').never()
 
-    with pytest.raises(ValueError), module.Runtime_directory({}):
+    with pytest.raises(ValueError), module.Runtime_directory({}, repository_id='id'):
         pass
 
 
-def test_runtime_directory_falls_back_to_runtime_directory():
+def test_runtime_directory_falls_back_to_runtime_directory(monkeypatch):
+    flexmock(module).should_receive('Fixed_name_temporary_directory').and_return(
+        flexmock(cleanup=lambda: None)
+    )
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(None)
+    monkeypatch.delenv('XDG_RUNTIME_DIR', raising=False)
     flexmock(module).should_receive('resolve_systemd_directory').with_args(
         module.Systemd_directories.RUNTIME_DIRECTORY
     ).and_return(
@@ -128,26 +121,12 @@ def test_runtime_directory_falls_back_to_runtime_directory():
     )
     flexmock(module.os).should_receive('makedirs')
 
-    with module.Runtime_directory({}) as borgmatic_runtime_directory:
+    with module.Runtime_directory({}, repository_id='id') as borgmatic_runtime_directory:
-        assert borgmatic_runtime_directory == '/run/./borgmatic'
+        assert borgmatic_runtime_directory == '/run/borgmatic-id/./borgmatic'
 
 
-def test_runtime_directory_falls_back_to_runtime_directory_without_adding_duplicate_borgmatic_subdirectory():
+def test_runtime_directory_with_relative_runtime_directory_errors(monkeypatch):
-    flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
+    monkeypatch.delenv('XDG_RUNTIME_DIR', raising=False)
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(None)
-    flexmock(module).should_receive('resolve_systemd_directory').with_args(
-        module.Systemd_directories.RUNTIME_DIRECTORY
-    ).and_return(
-        '/run/borgmatic',
-    )
-    flexmock(module.os).should_receive('makedirs')
-
-    with module.Runtime_directory({}) as borgmatic_runtime_directory:
-        assert borgmatic_runtime_directory == '/run/./borgmatic'
-
-
-def test_runtime_directory_with_relative_runtime_directory_errors():
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(None)
     flexmock(module).should_receive('resolve_systemd_directory').with_args(
         module.Systemd_directories.RUNTIME_DIRECTORY
     ).and_return(
@@ -155,19 +134,21 @@ def test_runtime_directory_with_relative_runtime_directory_errors():
     )
     flexmock(module.os).should_receive('makedirs').never()
 
-    with pytest.raises(ValueError), module.Runtime_directory({}):
+    with pytest.raises(ValueError), module.Runtime_directory({}, repository_id='id'):
         pass
 
 
-def test_runtime_directory_falls_back_to_tmpdir_and_adds_temporary_subdirectory_that_get_cleaned_up():
+def test_runtime_directory_falls_back_to_tmpdir_and_adds_temporary_subdirectory_that_get_cleaned_up(
+    monkeypatch,
+):
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(None)
+    monkeypatch.delenv('XDG_RUNTIME_DIR', raising=False)
     flexmock(module).should_receive('resolve_systemd_directory').with_args(
         module.Systemd_directories.RUNTIME_DIRECTORY
     ).and_return(
         None,
     )
-    flexmock(module.os.environ).should_receive('get').with_args('TMPDIR').and_return('/run')
+    monkeypatch.setenv('TMPDIR', '/run')
     temporary_directory = flexmock(name='/run/borgmatic-1234')
     temporary_directory.should_receive('cleanup').once()
     flexmock(module.tempfile).should_receive('TemporaryDirectory').with_args(
@@ -176,35 +157,37 @@ def test_runtime_directory_falls_back_to_tmpdir_and_adds_temporary_subdirectory_
     ).and_return(temporary_directory)
     flexmock(module.os).should_receive('makedirs')
 
-    with module.Runtime_directory({}) as borgmatic_runtime_directory:
+    with module.Runtime_directory({}, repository_id='id') as borgmatic_runtime_directory:
         assert borgmatic_runtime_directory == '/run/borgmatic-1234/./borgmatic'
 
 
-def test_runtime_directory_with_relative_tmpdir_errors():
+def test_runtime_directory_with_relative_tmpdir_errors(monkeypatch):
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(None)
+    monkeypatch.delenv('XDG_RUNTIME_DIR', raising=False)
     flexmock(module).should_receive('resolve_systemd_directory').with_args(
         module.Systemd_directories.RUNTIME_DIRECTORY
     ).and_return(
         None,
     )
-    flexmock(module.os.environ).should_receive('get').with_args('TMPDIR').and_return('run')
+    monkeypatch.setenv('TMPDIR', 'run')
     flexmock(module.tempfile).should_receive('TemporaryDirectory').never()
     flexmock(module.os).should_receive('makedirs').never()
 
-    with pytest.raises(ValueError), module.Runtime_directory({}):
+    with pytest.raises(ValueError), module.Runtime_directory({}, repository_id='id'):
         pass
 
 
-def test_runtime_directory_falls_back_to_temp_and_adds_temporary_subdirectory_that_get_cleaned_up():
+def test_runtime_directory_falls_back_to_temp_and_adds_temporary_subdirectory_that_get_cleaned_up(
+    monkeypatch,
+):
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(None)
+    monkeypatch.delenv('XDG_RUNTIME_DIR', raising=False)
     flexmock(module).should_receive('resolve_systemd_directory').with_args(
         module.Systemd_directories.RUNTIME_DIRECTORY
     ).and_return(
         None,
     )
-    flexmock(module.os.environ).should_receive('get').with_args('TMPDIR').and_return(None)
+    monkeypatch.delenv('TMPDIR', raising=False)
-    flexmock(module.os.environ).should_receive('get').with_args('TEMP').and_return('/run')
+    monkeypatch.setenv('TEMP', '/run')
     temporary_directory = flexmock(name='/run/borgmatic-1234')
     temporary_directory.should_receive('cleanup').once()
     flexmock(module.tempfile).should_receive('TemporaryDirectory').with_args(
@@ -213,36 +196,38 @@ def test_runtime_directory_falls_back_to_temp_and_adds_temporary_subdirectory_th
     ).and_return(temporary_directory)
     flexmock(module.os).should_receive('makedirs')
 
-    with module.Runtime_directory({}) as borgmatic_runtime_directory:
+    with module.Runtime_directory({}, repository_id='id') as borgmatic_runtime_directory:
         assert borgmatic_runtime_directory == '/run/borgmatic-1234/./borgmatic'
 
 
-def test_runtime_directory_with_relative_temp_errors():
+def test_runtime_directory_with_relative_temp_errors(monkeypatch):
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(None)
+    monkeypatch.delenv('XDG_RUNTIME_DIR', raising=False)
     flexmock(module).should_receive('resolve_systemd_directory').with_args(
         module.Systemd_directories.RUNTIME_DIRECTORY
     ).and_return(
         None,
     )
-    flexmock(module.os.environ).should_receive('get').with_args('TMPDIR').and_return(None)
+    monkeypatch.delenv('TMPDIR', raising=False)
-    flexmock(module.os.environ).should_receive('get').with_args('TEMP').and_return('run')
+    monkeypatch.setenv('TEMP', 'run')
     flexmock(module.tempfile).should_receive('TemporaryDirectory').never()
     flexmock(module.os).should_receive('makedirs')
 
-    with pytest.raises(ValueError), module.Runtime_directory({}):
+    with pytest.raises(ValueError), module.Runtime_directory({}, repository_id='id'):
         pass
 
 
-def test_runtime_directory_falls_back_to_hard_coded_tmp_path_and_adds_temporary_subdirectory_that_get_cleaned_up():
+def test_runtime_directory_falls_back_to_hard_coded_tmp_path_and_adds_temporary_subdirectory_that_get_cleaned_up(
+    monkeypatch,
+):
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(None)
+    monkeypatch.delenv('XDG_RUNTIME_DIR', raising=False)
     flexmock(module).should_receive('resolve_systemd_directory').with_args(
         module.Systemd_directories.RUNTIME_DIRECTORY
     ).and_return(
         None,
     )
-    flexmock(module.os.environ).should_receive('get').with_args('TMPDIR').and_return(None)
+    monkeypatch.delenv('TMPDIR', raising=False)
-    flexmock(module.os.environ).should_receive('get').with_args('TEMP').and_return(None)
+    monkeypatch.delenv('TEMP', raising=False)
     temporary_directory = flexmock(name='/tmp/borgmatic-1234')
     temporary_directory.should_receive('cleanup').once()
     flexmock(module.tempfile).should_receive('TemporaryDirectory').with_args(
@@ -251,20 +236,20 @@ def test_runtime_directory_falls_back_to_hard_coded_tmp_path_and_adds_temporary_
     ).and_return(temporary_directory)
     flexmock(module.os).should_receive('makedirs')
 
-    with module.Runtime_directory({}) as borgmatic_runtime_directory:
+    with module.Runtime_directory({}, repository_id='id') as borgmatic_runtime_directory:
         assert borgmatic_runtime_directory == '/tmp/borgmatic-1234/./borgmatic'
 
 
-def test_runtime_directory_with_erroring_cleanup_does_not_raise():
+def test_runtime_directory_with_erroring_cleanup_does_not_raise(monkeypatch):
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_RUNTIME_DIR').and_return(None)
+    monkeypatch.delenv('XDG_RUNTIME_DIR', raising=False)
     flexmock(module).should_receive('resolve_systemd_directory').with_args(
         module.Systemd_directories.RUNTIME_DIRECTORY
     ).and_return(
         None,
     )
-    flexmock(module.os.environ).should_receive('get').with_args('TMPDIR').and_return(None)
+    monkeypatch.delenv('TMPDIR', raising=False)
-    flexmock(module.os.environ).should_receive('get').with_args('TEMP').and_return(None)
+    monkeypatch.delenv('TEMP', raising=False)
     temporary_directory = flexmock(name='/tmp/borgmatic-1234')
     temporary_directory.should_receive('cleanup').and_raise(OSError).once()
     flexmock(module.tempfile).should_receive('TemporaryDirectory').with_args(
@@ -273,7 +258,7 @@ def test_runtime_directory_with_erroring_cleanup_does_not_raise():
     ).and_return(temporary_directory)
     flexmock(module.os).should_receive('makedirs')
 
-    with module.Runtime_directory({}) as borgmatic_runtime_directory:
+    with module.Runtime_directory({}, repository_id='id') as borgmatic_runtime_directory:
         assert borgmatic_runtime_directory == '/tmp/borgmatic-1234/./borgmatic'
 
 
@@ -291,7 +276,6 @@ def test_make_runtime_directory_glob(borgmatic_runtime_directory, expected_glob)
 
 def test_get_borgmatic_state_directory_uses_config_option():
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os.environ).should_receive('get').never()
 
     assert (
         module.get_borgmatic_state_directory(
@@ -301,16 +285,16 @@ def test_get_borgmatic_state_directory_uses_config_option():
     )
 
 
-def test_get_borgmatic_state_directory_falls_back_to_xdg_state_home():
+def test_get_borgmatic_state_directory_falls_back_to_xdg_state_home(monkeypatch):
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_STATE_HOME').and_return('/tmp')
+    monkeypatch.setenv('XDG_STATE_HOME', '/tmp')
 
     assert module.get_borgmatic_state_directory({}) == '/tmp/borgmatic'
 
 
-def test_get_borgmatic_state_directory_falls_back_to_state_directory():
+def test_get_borgmatic_state_directory_falls_back_to_state_directory(monkeypatch):
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os.environ).should_receive('get').with_args('XDG_STATE_HOME').and_return(None)
+    monkeypatch.delenv('XDG_STATE_HOME', raising=False)
     flexmock(module).should_receive('resolve_systemd_directory').with_args(
         module.Systemd_directories.STATE_DIRECTORY
     ).and_return(
@@ -320,32 +304,26 @@ def test_get_borgmatic_state_directory_falls_back_to_state_directory():
     assert module.get_borgmatic_state_directory({}) == '/tmp/borgmatic'
 
 
-def test_get_borgmatic_state_directory_defaults_to_hard_coded_path():
+def test_get_borgmatic_state_directory_defaults_to_hard_coded_path(monkeypatch):
     flexmock(module).should_receive('expand_user_in_path').replace_with(lambda path: path)
-    flexmock(module.os.environ).should_receive('get').and_return(None)
+    monkeypatch.delenv('XDG_STATE_HOME', raising=False)
     assert module.get_borgmatic_state_directory({}) == '~/.local/state/borgmatic'
 
 
-def test_resolve_systemd_directory_none():
+def test_resolve_systemd_directory_none(monkeypatch):
-    flexmock(module.os.environ).should_receive('get').with_args('RUNTIME_DIRECTORY').and_return(
+    monkeypatch.delenv('RUNTIME_DIRECTORY', raising=False)
-        None
+    monkeypatch.delenv('STATE_DIRECTORY', raising=False)
-    )
-    flexmock(module.os.environ).should_receive('get').with_args('STATE_DIRECTORY').and_return(None)
 
     assert module.resolve_systemd_directory(module.Systemd_directories.RUNTIME_DIRECTORY) is None
     assert module.resolve_systemd_directory(module.Systemd_directories.STATE_DIRECTORY) is None
 
 
-def test_resolve_systemd_directory_single():
+def test_resolve_systemd_directory_single(monkeypatch):
     runtime_dir = '/run/borgmatic'
     state_dir = '/var/lib/borgmatic'
 
-    flexmock(module.os.environ).should_receive('get').with_args('RUNTIME_DIRECTORY').and_return(
+    monkeypatch.setenv('RUNTIME_DIRECTORY', runtime_dir)
-        runtime_dir
+    monkeypatch.setenv('STATE_DIRECTORY', state_dir)
-    )
-    flexmock(module.os.environ).should_receive('get').with_args('STATE_DIRECTORY').and_return(
-        state_dir
-    )
 
     assert (
         module.resolve_systemd_directory(module.Systemd_directories.RUNTIME_DIRECTORY)
@@ -354,16 +332,12 @@ def test_resolve_systemd_directory_single():
     assert module.resolve_systemd_directory(module.Systemd_directories.STATE_DIRECTORY) == state_dir
 
 
-def test_resolve_systemd_directory_multiple():
+def test_resolve_systemd_directory_multiple(monkeypatch):
     runtime_dirs = '/run/borgmatic:/run/second:/run/third'
     state_dirs = '/var/lib/borgmatic:/var/lib/second:/var/lib/third'
 
-    flexmock(module.os.environ).should_receive('get').with_args('RUNTIME_DIRECTORY').and_return(
+    monkeypatch.setenv('RUNTIME_DIRECTORY', runtime_dirs)
-        runtime_dirs
+    monkeypatch.setenv('STATE_DIRECTORY', state_dirs)
-    )
-    flexmock(module.os.environ).should_receive('get').with_args('STATE_DIRECTORY').and_return(
-        state_dirs
-    )
 
     assert (
         module.resolve_systemd_directory(module.Systemd_directories.RUNTIME_DIRECTORY)