Fix exception when fetching notary server's old keys (#6625)

Lift the restriction that *all* the keys used for signing v2 key responses be
present in verify_keys.

Fixes #6596.

changelog.d/6625.bugfix

@@ -0,0 +1 @@
+Fix exception when fetching the `matrix.org:ed25519:auto` key.

synapse/crypto/keyring.py

@@ -511,17 +511,18 @@ class BaseV2KeyFetcher(object):
         server_name = response_json["server_name"]
         verified = False
         for key_id in response_json["signatures"].get(server_name, {}):
-            # each of the keys used for the signature must be present in the response
-            # json.
             key = verify_keys.get(key_id)
             if not key:
-                raise KeyLookupError(
+                # the key may not be present in verify_keys if:
-                    "Key response is signed by key id %s:%s but that key is not "
+                #  * we got the key from the notary server, and:
-                    "present in the response" % (server_name, key_id)
+                #  * the key belongs to the notary server, and:
-                )
+                #  * the notary server is using a different key to sign notary
+                #    responses.
+                continue
 
             verify_signed_json(response_json, server_name, key.verify_key)
             verified = True
+            break
 
         if not verified:
             raise KeyLookupError(

tests/crypto/test_keyring.py

@@ -19,6 +19,7 @@ from mock import Mock
 import canonicaljson
 import signedjson.key
 import signedjson.sign
+from nacl.signing import SigningKey
 from signedjson.key import encode_verify_key_base64, get_verify_key
 
 from twisted.internet import defer
@@ -412,6 +413,49 @@ class PerspectivesKeyFetcherTestCase(unittest.HomeserverTestCase):
             handlers=None, http_client=self.http_client, config=config
         )
 
+    def build_perspectives_response(
+        self, server_name: str, signing_key: SigningKey, valid_until_ts: int,
+    ) -> dict:
+        """
+        Build a valid perspectives server response to a request for the given key
+        """
+        verify_key = signedjson.key.get_verify_key(signing_key)
+        verifykey_id = "%s:%s" % (verify_key.alg, verify_key.version)
+
+        response = {
+            "server_name": server_name,
+            "old_verify_keys": {},
+            "valid_until_ts": valid_until_ts,
+            "verify_keys": {
+                verifykey_id: {
+                    "key": signedjson.key.encode_verify_key_base64(verify_key)
+                }
+            },
+        }
+        # the response must be signed by both the origin server and the perspectives
+        # server.
+        signedjson.sign.sign_json(response, server_name, signing_key)
+        self.mock_perspective_server.sign_response(response)
+        return response
+
+    def expect_outgoing_key_query(
+        self, expected_server_name: str, expected_key_id: str, response: dict
+    ) -> None:
+        """
+        Tell the mock http client to expect a perspectives-server key query
+        """
+
+        def post_json(destination, path, data, **kwargs):
+            self.assertEqual(destination, self.mock_perspective_server.server_name)
+            self.assertEqual(path, "/_matrix/key/v2/query")
+
+            # check that the request is for the expected key
+            q = data["server_keys"]
+            self.assertEqual(list(q[expected_server_name].keys()), [expected_key_id])
+            return {"server_keys": [response]}
+
+        self.http_client.post_json.side_effect = post_json
+
     def test_get_keys_from_perspectives(self):
         # arbitrarily advance the clock a bit
         self.reactor.advance(100)
@@ -424,33 +468,61 @@ class PerspectivesKeyFetcherTestCase(unittest.HomeserverTestCase):
         testverifykey_id = "ed25519:ver1"
         VALID_UNTIL_TS = 200 * 1000
 
-        # valid response
+        response = self.build_perspectives_response(
-        response = {
+            SERVER_NAME, testkey, VALID_UNTIL_TS,
-            "server_name": SERVER_NAME,
+        )
-            "old_verify_keys": {},
-            "valid_until_ts": VALID_UNTIL_TS,
-            "verify_keys": {
-                testverifykey_id: {
-                    "key": signedjson.key.encode_verify_key_base64(testverifykey)
-                }
-            },
-        }
 
-        # the response must be signed by both the origin server and the perspectives
+        self.expect_outgoing_key_query(SERVER_NAME, "key1", response)
-        # server.
-        signedjson.sign.sign_json(response, SERVER_NAME, testkey)
-        self.mock_perspective_server.sign_response(response)
 
-        def post_json(destination, path, data, **kwargs):
+        keys_to_fetch = {SERVER_NAME: {"key1": 0}}
-            self.assertEqual(destination, self.mock_perspective_server.server_name)
+        keys = self.get_success(fetcher.get_keys(keys_to_fetch))
-            self.assertEqual(path, "/_matrix/key/v2/query")
+        self.assertIn(SERVER_NAME, keys)
+        k = keys[SERVER_NAME][testverifykey_id]
+        self.assertEqual(k.valid_until_ts, VALID_UNTIL_TS)
+        self.assertEqual(k.verify_key, testverifykey)
+        self.assertEqual(k.verify_key.alg, "ed25519")
+        self.assertEqual(k.verify_key.version, "ver1")
 
-            # check that the request is for the expected key
+        # check that the perspectives store is correctly updated
-            q = data["server_keys"]
+        lookup_triplet = (SERVER_NAME, testverifykey_id, None)
-            self.assertEqual(list(q[SERVER_NAME].keys()), ["key1"])
+        key_json = self.get_success(
-            return {"server_keys": [response]}
+            self.hs.get_datastore().get_server_keys_json([lookup_triplet])
+        )
+        res = key_json[lookup_triplet]
+        self.assertEqual(len(res), 1)
+        res = res[0]
+        self.assertEqual(res["key_id"], testverifykey_id)
+        self.assertEqual(res["from_server"], self.mock_perspective_server.server_name)
+        self.assertEqual(res["ts_added_ms"], self.reactor.seconds() * 1000)
+        self.assertEqual(res["ts_valid_until_ms"], VALID_UNTIL_TS)
 
-        self.http_client.post_json.side_effect = post_json
+        self.assertEqual(
+            bytes(res["key_json"]), canonicaljson.encode_canonical_json(response)
+        )
+
+    def test_get_perspectives_own_key(self):
+        """Check that we can get the perspectives server's own keys
+
+        This is slightly complicated by the fact that the perspectives server may
+        use different keys for signing notary responses.
+        """
+
+        # arbitrarily advance the clock a bit
+        self.reactor.advance(100)
+
+        fetcher = PerspectivesKeyFetcher(self.hs)
+
+        SERVER_NAME = self.mock_perspective_server.server_name
+        testkey = signedjson.key.generate_signing_key("ver1")
+        testverifykey = signedjson.key.get_verify_key(testkey)
+        testverifykey_id = "ed25519:ver1"
+        VALID_UNTIL_TS = 200 * 1000
+
+        response = self.build_perspectives_response(
+            SERVER_NAME, testkey, VALID_UNTIL_TS
+        )
+
+        self.expect_outgoing_key_query(SERVER_NAME, "key1", response)
 
         keys_to_fetch = {SERVER_NAME: {"key1": 0}}
         keys = self.get_success(fetcher.get_keys(keys_to_fetch))
@@ -490,35 +562,14 @@ class PerspectivesKeyFetcherTestCase(unittest.HomeserverTestCase):
         VALID_UNTIL_TS = 200 * 1000
 
         def build_response():
-            # valid response
+            return self.build_perspectives_response(
-            response = {
+                SERVER_NAME, testkey, VALID_UNTIL_TS
-                "server_name": SERVER_NAME,
+            )
-                "old_verify_keys": {},
-                "valid_until_ts": VALID_UNTIL_TS,
-                "verify_keys": {
-                    testverifykey_id: {
-                        "key": signedjson.key.encode_verify_key_base64(testverifykey)
-                    }
-                },
-            }
-
-            # the response must be signed by both the origin server and the perspectives
-            # server.
-            signedjson.sign.sign_json(response, SERVER_NAME, testkey)
-            self.mock_perspective_server.sign_response(response)
-            return response
 
         def get_key_from_perspectives(response):
             fetcher = PerspectivesKeyFetcher(self.hs)
             keys_to_fetch = {SERVER_NAME: {"key1": 0}}
-
+            self.expect_outgoing_key_query(SERVER_NAME, "key1", response)
-            def post_json(destination, path, data, **kwargs):
-                self.assertEqual(destination, self.mock_perspective_server.server_name)
-                self.assertEqual(path, "/_matrix/key/v2/query")
-                return {"server_keys": [response]}
-
-            self.http_client.post_json.side_effect = post_json
-
             return self.get_success(fetcher.get_keys(keys_to_fetch))
 
         # start with a valid response so we can check we are testing the right thing