❇️ HTTP/2 with prior knowledge, and HTTP/3 with prior knowledge throu…

…gh svn toggles (#126)

It was already possible to force HTTP/3 using a fake entry in the quic
preemptive cache, but now you can leverage the main protocol toggles to
do it.

CHANGES.rst

@@ -1,3 +1,12 @@
+2.8.900 (2024-06-24)
+====================
+
+- Support for HTTP/2 with prior knowledge over non-encrypted connection to leverage multiplexing in internal networks.
+  To leverage this feature, you have to disable HTTP/1.1 so that `urllib3-future` can infer your intent.
+  Disabling HTTP/1.1 is to be made as follow: ``PoolManager(disabled_svn={HttpVersion.h11})``.
+- Added raw data bytes counter in ``LowLevelResponse`` to help end-users track download speed accordingly if they use
+  brotli, gzip or zstd transfer-encoding during downloads.
+
 2.7.914 (2024-06-15)
 ====================
 

README.md

@@ -29,6 +29,7 @@
 - HTTP/1.1, HTTP/2 and HTTP/3 support.
 - Proxy support for HTTP and SOCKS.
 - Detailed connection inspection.
+- HTTP/2 with prior knowledge.
 - Multiplexed connection.
 - Mirrored Sync & Async.
 - Amazingly Fast.
@@ -84,7 +85,18 @@ Or...
 import urllib3_future
 ```
 
-Doing `import urllib3_future` is the safest option for you as there is a significant number of projects that
+Or... upgrade any of your containers with...
+
+```dockerfile
+FROM python:3.12
+
+# ... your installation ...
+RUN pip install .
+# then! (after every other pip call)
+RUN pip install urllib3-future
+```
+
+Doing `import urllib3_future` is the safest option if you start a project from scratch for you as there is a significant number of projects that
 require `urllib3`.
 
 ## Notes

docs/advanced-usage.rst

@@ -1473,3 +1473,23 @@ This will enable up to 10 concurrent connections. To be clear, with that setting
 if your DNS resolver yield 6 addresses, you will spawn 6 tasks.
 
 .. warning:: Setting more than 20 is impracticable, DNS servers have a set limit of how many records can be returned. Most of the time, regular user are advised to leave the default value.
+
+HTTP/2 with prior knowledge
+---------------------------
+
+.. note:: Available since version 2.8+
+
+In some cases, mostly internal networks, you may desire to leverage multiplexing within a single HTTP connection without
+bothering with TLS (ALPN extensions) to discover and use HTTP/2 capabilities.
+
+You're in luck! urllib3-future now support talking with HTTP/2 server over an unencrypted connection.
+The only things you have to do is to disable HTTP/1.1 so that we can infer that you want to negotiate HTTP/2
+without any prior clear indicative that the remote can.
+
+Here is a simple example::
+
+    import urllib3
+
+    with urllib3.PoolManager(disabled_svn={urllib3.HttpVersion.h11}) as pm:
+        r = pm.urlopen("GET", "http://my-internal.svc.local/")
+

src/urllib3/_async/connection.py

@@ -737,7 +737,7 @@ async def connect(self) -> None:
                 tls_in_tls=tls_in_tls,
                 assert_hostname=self.assert_hostname,
                 assert_fingerprint=self.assert_fingerprint,
-                alpn_protocols=alpn_protocols,
+                alpn_protocols=alpn_protocols or None,
                 cert_data=self.cert_data,
                 key_data=self.key_data,
             )

src/urllib3/_version.py

@@ -1,4 +1,4 @@
 # This file is protected via CODEOWNERS
 from __future__ import annotations
 
-__version__ = "2.7.914"
+__version__ = "2.8.900"

src/urllib3/backend/_async/_base.py

@@ -63,6 +63,9 @@ def __init__(
             content_length = self.msg.get("content-length")
             self.length = int(content_length) if content_length else None
 
+        #: not part of http.client but useful to track (raw) download speeds!
+        self.data_in_count = 0
+
         self._stream_id = stream_id
 
         self.__buffer_excess: bytes = b""
@@ -131,12 +134,16 @@ async def read(self, __size: int | None = None) -> bytes:
         if self._eot and len(self.__buffer_excess) == 0:
             self.closed = True
 
+        size_in = len(data)
+
         if self.chunked:
             self.chunk_left = (
                 len(self.__buffer_excess) if self.__buffer_excess else None
             )
         elif self.length is not None:
-            self.length -= len(data)
+            self.length -= size_in
+
+        self.data_in_count += size_in
 
         return data
 

src/urllib3/backend/_async/hface.py

@@ -140,6 +140,13 @@ async def _new_conn(self) -> AsyncSocket | None:  # type: ignore[override]
                     self._svn = HttpVersion.h3
                     # we ignore alt-host as we do not trust cache security
                     self.port: int = self.__alt_authority[1]
+            elif (
+                HttpVersion.h11 in self._disabled_svn
+                and HttpVersion.h2 in self._disabled_svn
+            ):
+                self.__alt_authority = (self.host, self.port or 443)
+                self._svn = HttpVersion.h3
+                self.port = self.__alt_authority[1]
 
         if self._svn == HttpVersion.h3:
             self.socket_kind = SOCK_DGRAM
@@ -289,19 +296,51 @@ async def _post_conn(self) -> None:  # type: ignore[override]
 
         # first request was not made yet
         if self._svn is None:
-            if isinstance(
-                self.sock, (SSLAsyncSocket,)
-            ):  # TODO: Complete and verify this section
-                alpn: str | None = self.sock.selected_alpn_protocol()
+            # if we are on a TLS connection, inspect ALPN.
+            is_tcp_tls_conn = isinstance(self.sock, SSLAsyncSocket)
+
+            if is_tcp_tls_conn:
+                alpn: str | None = self.sock.selected_alpn_protocol()  # type: ignore[attr-defined]
 
                 if alpn is not None:
                     if alpn == "h2":
                         self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
                         self._svn = HttpVersion.h2
-                    elif alpn != "http/1.1":
+                    elif alpn == "http/1.1":
+                        self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
+                        self._svn = HttpVersion.h11
+                    else:
                         raise ProtocolError(  # Defensive: This should be unreachable as ALPN is explicit higher in the stack.
-                            f"Unsupported ALPN '{alpn}' during handshake"
+                            f"Unsupported ALPN '{alpn}' during handshake. Did you try to reach a non-HTTP server ?"
                         )
+                else:
+                    # no-alpn, let's decide between H2 or H11
+                    # by default, try HTTP/1.1
+                    if HttpVersion.h11 not in self._disabled_svn:
+                        self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
+                        self._svn = HttpVersion.h11
+                    elif HttpVersion.h2 not in self._disabled_svn:
+                        self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
+                        self._svn = HttpVersion.h2
+                    else:
+                        raise RuntimeError(
+                            "No compatible protocol are enabled to emit request. You currently are connected using "
+                            "TCP TLS and must have HTTP/1.1 or/and HTTP/2 enabled to pursue."
+                        )
+            else:
+                # no-TLS, let's decide between H2 or H11
+                # by default, try HTTP/1.1
+                if HttpVersion.h11 not in self._disabled_svn:
+                    self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
+                    self._svn = HttpVersion.h11
+                elif HttpVersion.h2 not in self._disabled_svn:
+                    self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
+                    self._svn = HttpVersion.h2
+                else:
+                    raise RuntimeError(
+                        "No compatible protocol are enabled to emit request. You currently are connected using "
+                        "TCP Unencrypted and must have HTTP/1.1 or/and HTTP/2 enabled to pursue."
+                    )
         else:
             if self._svn == HttpVersion.h2:
                 self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
@@ -392,8 +431,13 @@ async def _post_conn(self) -> None:  # type: ignore[override]
 
         # fallback to http/1.1
         if self._protocol is None or self._svn == HttpVersion.h11:
-            self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
-            self._svn = HttpVersion.h11
+            if self._protocol is None and HttpVersion.h11 in self._disabled_svn:
+                self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
+                self._svn = HttpVersion.h2
+            else:
+                self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
+                self._svn = HttpVersion.h11
+
             self.conn_info.http_version = self._svn
 
             if (

src/urllib3/backend/_base.py

@@ -138,6 +138,9 @@ def __init__(
             content_length = self.msg.get("content-length")
             self.length = int(content_length) if content_length else None
 
+        #: not part of http.client but useful to track (raw) download speeds!
+        self.data_in_count = 0
+
         # tricky part...
         # sometime 3rd party library tend to access hazardous materials...
         # they want a direct socket access.
@@ -231,12 +234,16 @@ def read(self, __size: int | None = None) -> bytes:
             self.closed = True
             self._sock = None
 
+        size_in = len(data)
+
         if self.chunked:
             self.chunk_left = (
                 len(self.__buffer_excess) if self.__buffer_excess else None
             )
         elif self.length is not None:
-            self.length -= len(data)
+            self.length -= size_in
+
+        self.data_in_count += size_in
 
         return data
 
@@ -371,9 +378,9 @@ def __init__(
         self._preemptive_quic_cache = preemptive_quic_cache
 
         if self._disabled_svn:
-            if HttpVersion.h11 in self._disabled_svn:
+            if len(self._disabled_svn) == len(list(HttpVersion)):
                 raise RuntimeError(
-                    "HTTP/1.1 cannot be disabled. It will be allowed in a future major version."
+                    "You disabled every supported protocols. The HTTP connection object is left with no outcomes."
                 )
 
         # valuable intel

src/urllib3/backend/hface.py

@@ -154,6 +154,13 @@ def _new_conn(self) -> socket.socket | None:
                     self._svn = HttpVersion.h3
                     # we ignore alt-host as we do not trust cache security
                     self.port: int = self.__alt_authority[1]
+            elif (
+                HttpVersion.h11 in self._disabled_svn
+                and HttpVersion.h2 in self._disabled_svn
+            ):
+                self.__alt_authority = (self.host, self.port or 443)
+                self._svn = HttpVersion.h3
+                self.port = self.__alt_authority[1]
 
         if self._svn == HttpVersion.h3:
             self.socket_kind = SOCK_DGRAM
@@ -311,9 +318,12 @@ def _post_conn(self) -> None:
             self.sock is not None
         ), "probable attempt to call _post_conn() prior to successful _new_conn()"
 
-        # first request was not made yet
+        # first request was not made yet // need to infer what protocol to use.
         if self._svn is None:
-            if isinstance(self.sock, (ssl.SSLSocket, SSLTransport)):
+            # if we are on a TLS connection, inspect ALPN.
+            is_tcp_tls_conn = isinstance(self.sock, (ssl.SSLSocket, SSLTransport))
+
+            if is_tcp_tls_conn:
                 alpn: str | None = (
                     self.sock.selected_alpn_protocol()
                     if isinstance(self.sock, ssl.SSLSocket)
@@ -324,11 +334,42 @@ def _post_conn(self) -> None:
                     if alpn == "h2":
                         self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
                         self._svn = HttpVersion.h2
-                    elif alpn != "http/1.1":
+                    elif alpn == "http/1.1":
+                        self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
+                        self._svn = HttpVersion.h11
+                    else:
                         raise ProtocolError(  # Defensive: This should be unreachable as ALPN is explicit higher in the stack.
-                            f"Unsupported ALPN '{alpn}' during handshake"
+                            f"Unsupported ALPN '{alpn}' during handshake. Did you try to reach a non-HTTP server ?"
                         )
-        else:
+                else:
+                    # no-alpn, let's decide between H2 or H11
+                    # by default, try HTTP/1.1
+                    if HttpVersion.h11 not in self._disabled_svn:
+                        self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
+                        self._svn = HttpVersion.h11
+                    elif HttpVersion.h2 not in self._disabled_svn:
+                        self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
+                        self._svn = HttpVersion.h2
+                    else:
+                        raise RuntimeError(
+                            "No compatible protocol are enabled to emit request. You currently are connected using "
+                            "TCP TLS and must have HTTP/1.1 or/and HTTP/2 enabled to pursue."
+                        )
+            else:
+                # no-TLS, let's decide between H2 or H11
+                # by default, try HTTP/1.1
+                if HttpVersion.h11 not in self._disabled_svn:
+                    self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
+                    self._svn = HttpVersion.h11
+                elif HttpVersion.h2 not in self._disabled_svn:
+                    self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
+                    self._svn = HttpVersion.h2
+                else:
+                    raise RuntimeError(
+                        "No compatible protocol are enabled to emit request. You currently are connected using "
+                        "TCP Unencrypted and must have HTTP/1.1 or/and HTTP/2 enabled to pursue."
+                    )
+        else:  # we or someone manually set the SVN / http version, so load the protocol regardless of what we know.
             if self._svn == HttpVersion.h2:
                 self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
             elif self._svn == HttpVersion.h3:
@@ -359,6 +400,7 @@ def _post_conn(self) -> None:
             self.conn_info.resolution_latency = self._connect_timings[0]
             self.conn_info.established_latency = self._connect_timings[1]
 
+        #: Populating the ConnectionInfo using Python native capabilities
         if self._svn != HttpVersion.h3:
             cipher_tuple: tuple[str, str, int] | None = None
 
@@ -448,10 +490,15 @@ def _post_conn(self) -> None:
             ):
                 self.conn_info.destination_address = self.sock.getpeername()[:2]
 
-        # fallback to http/1.1
+        # fallback to http/1.1 or http/2 with prior knowledge!
         if self._protocol is None or self._svn == HttpVersion.h11:
-            self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
-            self._svn = HttpVersion.h11
+            if self._protocol is None and HttpVersion.h11 in self._disabled_svn:
+                self._protocol = HTTPProtocolFactory.new(HTTP2Protocol)  # type: ignore[type-abstract]
+                self._svn = HttpVersion.h2
+            else:
+                self._protocol = HTTPProtocolFactory.new(HTTP1Protocol)  # type: ignore[type-abstract]
+                self._svn = HttpVersion.h11
+
             self.conn_info.http_version = self._svn
 
             if (
@@ -471,6 +518,7 @@ def _post_conn(self) -> None:
             receive_first=False,
         )
 
+        #: Populating ConnectionInfo using QUIC TLS interfaces
         if isinstance(self._protocol, HTTPOverQUICProtocol):
             self.conn_info.certificate_der = self._protocol.getpeercert(
                 binary_form=True

src/urllib3/connection.py

@@ -735,7 +735,7 @@ def connect(self) -> None:
                 tls_in_tls=tls_in_tls,
                 assert_hostname=self.assert_hostname,
                 assert_fingerprint=self.assert_fingerprint,
-                alpn_protocols=alpn_protocols,
+                alpn_protocols=alpn_protocols or None,
                 cert_data=self.cert_data,
                 key_data=self.key_data,
             )