🔖 Release 2.2.904 (#41)

2.2.904 (2023-11-06)
=================


- Fixed concurrent/multiplexed request overflow in a full connection
pool.
- Fixed connection close that had in-flight request (in multiplexed
mode), the connection appeared as not idle on clean reuse.

CHANGES.rst

@@ -1,3 +1,9 @@
+2.2.904 (2023-11-06)
+====================
+
+- Fixed concurrent/multiplexed request overflow in a full connection pool.
+- Fixed connection close that had in-flight request (in multiplexed mode), the connection appeared as not idle on clean reuse.
+
 2.2.903 (2023-11-06)
 ====================
 

src/urllib3/_version.py

@@ -1,4 +1,4 @@
 # This file is protected via CODEOWNERS
 from __future__ import annotations
 
-__version__ = "2.2.903"
+__version__ = "2.2.904"

src/urllib3/backend/_base.py

@@ -99,7 +99,9 @@ def from_promise(self) -> ResponsePromise | None:
     @from_promise.setter
     def from_promise(self, value: ResponsePromise) -> None:
         if value.stream_id != self._stream_id:
-            raise ValueError
+            raise ValueError(
+                "Trying to assign a ResponsePromise to an unrelated LowLevelResponse"
+            )
         self.__promise = value
 
     @property

src/urllib3/backend/hface.py

@@ -978,3 +978,9 @@ def close(self) -> None:
 
         self._protocol = None
         self._stream_id = None
+        self._promises = {}
+        self._pending_responses = {}
+        self.__custom_tls_settings = None
+        self.conn_info = None
+        self.__expected_body_length = None
+        self.__remaining_body_length = None

src/urllib3/connectionpool.py

@@ -196,6 +196,7 @@ def __init__(
         self.timeout = timeout
         self.retries = retries
 
+        self._maxsize = maxsize
         self.pool: queue.LifoQueue[typing.Any] | None = self.QueueCls(maxsize)
         self.block = block
 
@@ -284,7 +285,7 @@ def _get_conn(
             pass  # Oh well, we'll create a new connection then
 
         if no_new and conn is None:
-            raise ValueError
+            raise ValueError("Explicitly asked to stop get_conn early with no_new=True")
 
         # If this is a persistent connection, check if it got disconnected
         if conn and is_connection_dropped(conn):
@@ -322,14 +323,29 @@ def _put_conn(self, conn: HTTPConnection | None) -> None:
             except queue.Full:
                 # Connection never got put back into the pool, close it.
                 if conn:
-                    conn.close()
+                    if conn.is_idle:
+                        conn.close()
 
                 if self.block:
                     # This should never happen if you got the conn from self._get_conn
                     raise FullPoolError(
                         self,
                         "Pool reached maximum size and no more connections are allowed.",
                     ) from None
+                else:
+                    # multiplexed connection may still have in-flight request not converted into response
+                    # we shall not discard it until responses are consumed.
+                    if conn and conn.is_idle is False:
+                        log.warning(
+                            "Connection pool is full, temporary increase, keeping connection, "
+                            "multiplexed and not idle: %s. Connection pool size: %s",
+                            self.host,
+                            self.pool.qsize(),
+                        )
+
+                        self.pool.maxsize += 1
+
+                        return self._put_conn(conn)
 
                 log.warning(
                     "Connection pool is full, discarding connection: %s. Connection pool size: %s",
@@ -414,8 +430,24 @@ def get_response(
                     continue
             break
 
+        forget_about_connections = []
+
+        # we exceptionally increased the size (block=False + multiplexed enabled)
+        if len(connections) > self._maxsize:
+            expect_drop_count = len(connections) - self._maxsize
+
+            for conn in connections:
+                if conn.is_idle:
+                    forget_about_connections.append(conn)
+                if len(forget_about_connections) >= expect_drop_count:
+                    break
+
         for conn in connections:
-            self._put_conn(conn)
+            if conn not in forget_about_connections:
+                self._put_conn(conn)
+
+        for conn in forget_about_connections:
+            conn.close()
 
         if promise is not None and response is None:
             raise ValueError(
@@ -438,7 +470,9 @@ def get_response(
                 from_promise = response._fp.from_promise
 
         if from_promise is None:
-            raise ValueError
+            raise ValueError(
+                "Internal: Unable to identify originating ResponsePromise from a LowLevelResponse"
+            )
 
         # Retrieve request ctx
         method = typing.cast(str, from_promise.get_parameter("method"))

src/urllib3/poolmanager.py

@@ -475,7 +475,9 @@ def get_response(
                 from_promise = response._fp.from_promise
 
         if from_promise is None:
-            raise ValueError
+            raise ValueError(
+                "Internal: Unable to identify originating ResponsePromise from a LowLevelResponse"
+            )
 
         # Retrieve request ctx
         method = typing.cast(str, from_promise.get_parameter("method"))