feat: Support OS truststore for the TLS certificate verification

This commit add support for loading the OS truststore root certificates in addition to the "legacy" certifi bundle. It will come in handy for every user behind a company proxy or just using a private PyPI warehouse.

Close python-poetry#9249

docs/repositories.md

@@ -569,6 +569,11 @@ poetry config -- http-basic.pypi myUsername -myPasswordStartingWithDash
 
 ## Certificates
 
+### OS Truststore
+
+Poetry access the system truststore by default and retrieve the root certificates appropriately on Linux, Windows, and MacOS.
+In addition to your OS root certificates, we still load the authorities provided by `certifi` as before.
+
 ### Custom certificate authority and mutual TLS authentication
 
 Poetry supports repositories that are secured by a custom certificate authority as well as those that require

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "poetry"
-version = "1.9.0.dev0"
+version = "2.0.0.dev0"
 description = "Python dependency management and packaging made easy."
 authors = ["Sébastien Eustace <sebastien@eustace.io>"]
 maintainers = [
@@ -31,8 +31,7 @@ Changelog = "https://python-poetry.org/history/"
 [tool.poetry.dependencies]
 python = "^3.8"
 
-poetry-core = "1.9.0"
-poetry-plugin-export = "^1.8.0"
+poetry-core = { git = "https://github.com/python-poetry/poetry-core.git", branch = "main" }
 build = "^1.2.1"
 cachecontrol = { version = "^0.14.0", extras = ["filecache"] }
 cleo = "^2.1.0"
@@ -54,8 +53,9 @@ tomli = { version = "^2.0.1", python = "<3.11" }
 tomlkit = ">=0.11.4,<1.0.0"
 # trove-classifiers uses calver, so version is unclamped
 trove-classifiers = ">=2022.5.19"
-virtualenv = "^20.23.0"
+virtualenv = "^20.26.6"
 xattr = { version = "^1.0.0", markers = "sys_platform == 'darwin'" }
+wassima = "^1.1.3"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = ">=2.10"
@@ -126,7 +126,6 @@ extend-select = [
 ignore = [
     "B904", # use 'raise ... from err'
     "B905", # use explicit 'strict=' parameter with 'zip()'
-    "N818", #  Exception name should be named with an Error suffix
 ]
 extend-safe-fixes = [
     "TCH", # move import from and to TYPE_CHECKING blocks

src/poetry/publishing/uploader.py

@@ -18,6 +18,7 @@
 from poetry.publishing.hash_manager import HashManager
 from poetry.utils.constants import REQUESTS_TIMEOUT
 from poetry.utils.patterns import wheel_file_re
+from poetry.utils.truststore import WithTrustStoreAdapter
 
 
 if TYPE_CHECKING:
@@ -88,6 +89,9 @@ def auth(self, username: str | None, password: str | None) -> None:
 
     def make_session(self) -> requests.Session:
         session = requests.Session()
+        adapter = WithTrustStoreAdapter()
+        session.mount("http://", adapter)
+        session.mount("https://", adapter)
         auth = self.get_auth()
         if auth is not None:
             session.auth = auth

src/poetry/utils/authenticator.py

@@ -16,7 +16,6 @@
 import requests.auth
 import requests.exceptions
 
-from cachecontrol import CacheControlAdapter
 from cachecontrol.caches import FileCache
 from requests_toolbelt import user_agent
 
@@ -28,6 +27,7 @@
 from poetry.utils.constants import STATUS_FORCELIST
 from poetry.utils.password_manager import HTTPAuthCredential
 from poetry.utils.password_manager import PasswordManager
+from poetry.utils.truststore import CacheControlWithTrustStoreAdapter
 
 
 if TYPE_CHECKING:
@@ -137,7 +137,7 @@ def create_session(self) -> requests.Session:
         if self._cache_control is None:
             return session
 
-        adapter = CacheControlAdapter(
+        adapter = CacheControlWithTrustStoreAdapter(
             cache=self._cache_control,
             pool_maxsize=self._pool_size,
         )

src/poetry/utils/truststore.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import typing
+
+from cachecontrol import CacheControlAdapter
+from requests.adapters import HTTPAdapter
+from wassima import RUSTLS_LOADED
+from wassima import generate_ca_bundle
+
+
+if typing.TYPE_CHECKING:
+    from urllib3 import HTTPConnectionPool
+
+
+DEFAULT_CA_BUNDLE: str = generate_ca_bundle()
+
+
+class WithTrustStoreAdapter(HTTPAdapter):
+    """
+    Inject the OS truststore in Requests.
+    Certifi is still loaded in addition to the OS truststore for (strict) backward compatibility purposes.
+    See https://github.com/jawah/wassima for more details.
+    """
+
+    def cert_verify(
+        self,
+        conn: HTTPConnectionPool,
+        url: str,
+        verify: bool | str,
+        cert: str | tuple[str, str] | None,
+    ) -> None:
+        #: only apply truststore cert if "verify" is set with default value "True".
+        #: RUSTLS_LOADED means that "wassima" is not the py3 none wheel and does not fallback on "certifi"
+        #: if "RUSTLS_LOADED" is False then "wassima" just return "certifi" bundle instead.
+        if (
+            RUSTLS_LOADED
+            and url.lower().startswith("https")
+            and verify is True
+            and hasattr(conn, "ca_cert_data")
+        ):
+            # url starting with https already mean that conn is a HTTPSConnectionPool
+            # the hasattr is to make mypy happy.
+            conn.ca_cert_data = DEFAULT_CA_BUNDLE
+
+        # still apply upstream logic as before
+        super().cert_verify(conn, url, verify, cert)  # type: ignore[no-untyped-call]
+
+
+class CacheControlWithTrustStoreAdapter(CacheControlAdapter):
+    """
+    Same as WithTrustStoreAdapter but with CacheControlAdapter as its parent
+    class.
+    """
+
+    def cert_verify(
+        self,
+        conn: HTTPConnectionPool,
+        url: str,
+        verify: bool | str,
+        cert: str | tuple[str, str] | None,
+    ) -> None:
+        if (
+            RUSTLS_LOADED
+            and url.lower().startswith("https")
+            and verify is True
+            and hasattr(conn, "ca_cert_data")
+        ):
+            conn.ca_cert_data = DEFAULT_CA_BUNDLE
+
+        super().cert_verify(conn, url, verify, cert)  # type: ignore[no-untyped-call]

src/poetry/vcs/git/backend.py

@@ -11,8 +11,12 @@
 from urllib.parse import urlparse
 from urllib.parse import urlunparse
 
+import certifi
+import wassima
+
 from dulwich import porcelain
 from dulwich.client import HTTPUnauthorized
+from dulwich.client import default_urllib3_manager
 from dulwich.client import get_transport_and_path
 from dulwich.config import ConfigFile
 from dulwich.config import parse_submodules
@@ -204,6 +208,19 @@ def _fetch_remote_refs(cls, url: str, local: Repo) -> FetchPackResult:
             kwargs["password"] = credentials.password
 
         config = local.get_config_stack()
+
+        # we want to inject the system root CA if the transport is urllib3 (aka. http)
+        # so that our users won't need the "system" git.
+        # we combine the system trust store with the certifi bundle.
+        pool_manager = default_urllib3_manager(
+            config,
+            ca_certs=certifi.where(),
+            ca_cert_data=wassima.generate_ca_bundle()
+            if wassima.RUSTLS_LOADED
+            else None,
+        )
+        kwargs["pool_manager"] = pool_manager
+
         client, path = get_transport_and_path(url, config=config, **kwargs)
 
         with local: