diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py
index e348c387..c325454b 100644
--- a/borgmatic/actions/restore.py
+++ b/borgmatic/actions/restore.py
@@ -33,9 +33,9 @@ def dumps_match(first, second, default_port=None):
     indicates that the field should match any value. If a default port is given, then consider any
     dump having that port to match with a dump having a None port.
     '''
-    # label kinda counts as id, if they match, consider it a full match
+    # label kinda counts as id, if they match ignore hostname & port
     if first.label not in {None, UNSPECIFIED} and first.label == second.label:
-        return True
+        return first[:2] == second[:2]
 
     for field_name in first._fields:
         first_value = getattr(first, field_name)