Interfaces and services for JWK management

requirements/main.in

@@ -43,6 +43,7 @@ pyramid_rpc>=0.7
 pyramid_services>=2.1
 pyramid_tm>=0.12
 python-slugify
+PyJWT[crypto]>=2.3.0
 readme-renderer[md]>=0.7.0
 requests
 requests-aws4auth

requirements/main.txt

@@ -283,7 +283,8 @@ cryptography==36.0.1 \
     --hash=sha256:ebc15b1c22e55c4d5566e3ca4db8689470a0ca2babef8e3a9ee057a8b82ce4b1 \
     --hash=sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee
     # via
-    #   -r requirements/main.in
+    #   -r main.in
+    #   pyjwt
     #   pyopenssl
     #   webauthn
 cssselect==1.1.0 \
@@ -883,6 +884,10 @@ pygments==2.10.0 \
     --hash=sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380 \
     --hash=sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6
     # via readme-renderer
+pyjwt[crypto]==2.3.0 \
+    --hash=sha256:b888b4d56f06f6dcd777210c334e69c737be74755d3e5e9ee3fe67dc18a0ee41 \
+    --hash=sha256:e0c4bb8d9f0af0c7f5b1ec4c5036309617d03d56932877f2f7a0beeb5318322f
+    # via -r main.in
 pymacaroons==0.13.0 \
     --hash=sha256:1e6bba42a5f66c245adf38a5a4006a99dcc06a0703786ea636098667d42903b8 \
     --hash=sha256:3e14dff6a262fdbf1a15e769ce635a8aea72e6f8f91e408f9a97166c53b91907

tests/unit/test_config.py

@@ -332,6 +332,7 @@ def __init__(self):
             pretend.call(".email"),
             pretend.call(".accounts"),
             pretend.call(".macaroons"),
+            pretend.call(".oidc"),
             pretend.call(".malware"),
             pretend.call(".manage"),
             pretend.call(".packaging"),

warehouse/config.py

@@ -178,6 +178,7 @@ def configure(settings=None):
     maybe_set(settings, "celery.broker_url", "BROKER_URL")
     maybe_set(settings, "celery.result_url", "REDIS_URL")
     maybe_set(settings, "celery.scheduler_url", "REDIS_URL")
+    maybe_set(settings, "oidc.jwk_cache_url", "REDIS_URL")
     maybe_set(settings, "database.url", "DATABASE_URL")
     maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_URL")
     maybe_set(settings, "elasticsearch.url", "ELASTICSEARCH_SIX_URL")
@@ -458,6 +459,9 @@ def configure(settings=None):
     # Register support for Macaroon based authentication
     config.include(".macaroons")
 
+    # Register support for OIDC provider based authentication
+    config.include(".oidc")
+
     # Register support for malware checks
     config.include(".malware")
 

warehouse/oidc/__init__.py

@@ -0,0 +1,20 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from warehouse.oidc.interfaces import IJWKService
+from warehouse.oidc.services import JWKServiceFactory
+
+
+def includeme(config):
+    config.register_service_factory(
+        JWKServiceFactory(provider="github"), IJWKService, name="github"
+    )

warehouse/oidc/interfaces.py

@@ -0,0 +1,34 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from zope.interface import Interface
+
+
+class IJWKService(Interface):
+    def create_service(context, request):
+        """
+        Create the service, given the context and request for which it is
+        being created.
+        """
+        pass
+
+    def get_key(provider, key_id):
+        """
+        Return the JWK identified by the given KID for the given provider,
+        fetching it if not already cached locally.
+
+        Returns None if the JWK does not exist or the access pattern is
+        invalid (i.e., exceeds our internal limit on JWK requests to
+        each provider).
+        """
+        pass

warehouse/oidc/services.py

@@ -0,0 +1,168 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import redis
+import requests
+import sentry_sdk
+
+from jwt import PyJWK
+from zope.interface import implementer
+
+from warehouse.metrics.interfaces import IMetricsService
+from warehouse.oidc.interfaces import IJWKService
+from warehouse.utils import oidc
+
+
+@implementer(IJWKService)
+class JWKService:
+    def __init__(self, provider, cache_url, metrics):
+        self.provider
+        self.cache_url = cache_url
+        self.metrics = metrics
+
+        self._provider_jwk_key = f"/warehouse/oidc/jwks/{self.provider}"
+        self._provider_timeout_key = f"{self._provider_jwk_key}/timeout"
+
+    def _store_keyset(self, keys):
+        """
+        Store the given keyset for the given provider, setting the timeout key
+        in the process.
+        """
+
+        with redis.StrictRedis.from_url(self.cache_url) as r:
+            r.set(self._provider_jwk_key, json.dumps(keys))
+            r.setex(self._provider_timeout_key, 60, "placeholder")
+
+    def _get_keyset(self):
+        """
+        Return the cached keyset for the given provider, or an empty
+        keyset if no keys are currently cached.
+        """
+
+        with redis.StrictRedis.from_url(self.cache_url) as r:
+            keys = r.get(self._provider_jwk_key)
+            timeout = bool(r.exists(self._provider_timeout_key))
+            if keys is not None:
+                return (json.loads(keys), timeout)
+            else:
+                return ({}, timeout)
+
+    def _refresh_keyset(self):
+        """
+        Attempt to refresh the keyset from the OIDC provider, assuming no
+        timeout is in effect.
+
+        Returns the refreshed keyset, or the cached keyset if a timeout is
+        in effect.
+
+        Returns the cached keyset on any provider access or format errors.
+        """
+
+        # Fast path: we're in a cooldown from a previous refresh.
+        keys, timeout = self._get_keyset()
+        if timeout:
+            self.metrics.increment("warehouse.oidc.refresh_keyset.timeout")
+            return keys
+
+        oidc_url = f"{oidc.OIDC_PROVIDERS[self.provider]}/{oidc.WELL_KNOWN_OIDC_CONF}"
+
+        resp = requests.get(oidc_url)
+
+        # For whatever reason, an OIDC provider's configuration URL might be
+        # offline. We don't want to completely explode here, since other
+        # providers might still be online (and need updating), so we spit
+        # out an error and return None instead of raising.
+        if not resp.ok:
+            sentry_sdk.capture_message(
+                f"OIDC provider {self.provider} failed to return configuration: "
+                f"{oidc_url}"
+            )
+            return keys
+
+        oidc_conf = resp.json()
+        jwks_url = oidc_conf.get("jwks_uri")
+
+        # A valid OIDC configuration MUST have a `jwks_uri`, but we
+        # defend against its absence anyways.
+        if jwks_url is None:
+            sentry_sdk.capture_message(
+                f"OIDC provider {self.provider} is returning malformed "
+                "configuration (no jwks_uri)"
+            )
+            return keys
+
+        resp = requests.get(jwks_url)
+
+        # Same reasoning as above.
+        if not resp.ok:
+            sentry_sdk.capture_message(
+                f"OIDC provider {self.provider} failed to return JWKS JSON: "
+                f"{jwks_url}"
+            )
+            return keys
+
+        jwks_conf = resp.json()
+        keys = jwks_conf.get("keys")
+
+        # Another sanity test: an OIDC provider should never return an empty
+        # keyset, but there's nothing stopping them from doing so. We don't
+        # want to cache an empty keyset just in case it's a short-lived error,
+        # so we check here, error, and return the current cache instead.
+        if not keys:
+            sentry_sdk.capture_message(
+                f"OIDC provider {self.provider} returned JWKS but no keys"
+            )
+            return keys
+
+        keys = {key["kid"]: key for key in keys}
+        self._store_keyset(keys)
+
+        return keys
+
+    def get_key(self, key_id):
+        """
+        Return a JWK for the given key ID, or None if the key can't be found
+        in this provider's keyset.
+        """
+
+        keyset, _ = self._get_keyset()
+        if key_id not in keyset:
+            keyset = self._refresh_keyset()
+        if key_id not in keyset:
+            self.metrics.increment(
+                "warehouse.oidc.get_key.error", tags=[f"key_id:{key_id}"]
+            )
+            return None
+        return PyJWK(keyset[key_id])
+
+
+class JWKServiceFactory:
+    def __init__(self, provider, service_class=JWKService):
+        self.provider = provider
+        self.service_class = service_class
+
+    def __call__(self, _context, request):
+        cache_url = request.registry.settings["oidc.jwk_cache_url"]
+        metrics = request.find_service(IMetricsService, context=None)
+
+        return self.service_class(self.provider, cache_url, metrics)
+
+    def __eq__(self, other):
+        if not isinstance(other, JWKServiceFactory):
+            return NotImplemented
+
+        return (self.provider, self.service_class) == (
+            other.provider,
+            other.service_class,
+        )

warehouse/utils/oidc.py

@@ -0,0 +1,27 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# For now, only RS256 is supported.
+# We probably won't need to support providers with only symmetric keys
+# (e.g. HS256) in the foreseeable future.
+VALID_ALGS = {"RS256"}
+
+# This is a map of OIDC providers supported by Warehouse.
+# The keys are human-readable names for each provider, and the values
+# are the "issuer" (i.e., `iss`) FQDN that JWTs signed by their respective
+# JWKs are checked against.
+OIDC_PROVIDERS = {
+    "github": "https://token.actions.githubusercontent.com",
+}
+
+WELL_KNOWN_OIDC_CONF = ".well-known/openid-configuration"