From cfed75aedef68c97d682a9708c197ffe2287e260 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Mon, 21 Oct 2024 09:48:17 +0200 Subject: [PATCH 1/4] :bug: shielding errors in our picotls logic --- src/niquests/extensions/_async_ocsp.py | 20 +++++++++----- src/niquests/extensions/_ocsp.py | 19 ++++++++----- src/niquests/extensions/_picotls.py | 37 +++++++++++++++++++------- 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/src/niquests/extensions/_async_ocsp.py b/src/niquests/extensions/_async_ocsp.py index b089f67fb6..1aabd593a4 100644 --- a/src/niquests/extensions/_async_ocsp.py +++ b/src/niquests/extensions/_async_ocsp.py @@ -51,6 +51,7 @@ async_recv_tls, async_recv_tls_and_decrypt, async_send_tls, + PicoTLSException, ) from ._ocsp import ( _str_fingerprint_of, @@ -73,7 +74,11 @@ async def _ask_nicely_for_issuer( sock = AsyncSocket(socket.AF_INET6, socket.SOCK_STREAM) sock.settimeout(timeout) - await sock.connect(dst_address) + + try: + await sock.connect(dst_address) + except (OSError, socket.timeout, TimeoutError, ConnectionError) as e: + raise PicoTLSException from e SECP256R1_P = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF SECP256R1_A = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFC @@ -408,11 +413,14 @@ async def verify( raise ValueError if not proxies: - issuer_certificate = await _ask_nicely_for_issuer( - url_parsed.hostname, - conn_info.destination_address, - timeout, - ) + try: + issuer_certificate = await _ask_nicely_for_issuer( + url_parsed.hostname, + conn_info.destination_address, + timeout, + ) + except PicoTLSException: + issuer_certificate = None else: issuer_certificate = None diff --git a/src/niquests/extensions/_ocsp.py b/src/niquests/extensions/_ocsp.py index 1145ca8c8a..b9740c842c 100644 --- a/src/niquests/extensions/_ocsp.py +++ b/src/niquests/extensions/_ocsp.py @@ -49,6 +49,7 @@ recv_tls, recv_tls_and_decrypt, send_tls, + PicoTLSException, ) @@ -80,8 +81,11 @@ def _ask_nicely_for_issuer( else: sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - sock.connect(dst_address) sock.settimeout(timeout) + try: + sock.connect(dst_address) + except (OSError, socket.timeout, TimeoutError, ConnectionError) as e: + raise PicoTLSException from e SECP256R1_P = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF SECP256R1_A = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFC @@ -402,11 +406,14 @@ def verify( raise ValueError if not proxies: - issuer_certificate = _ask_nicely_for_issuer( - url_parsed.hostname, - conn_info.destination_address, - timeout, - ) + try: + issuer_certificate = _ask_nicely_for_issuer( + url_parsed.hostname, + conn_info.destination_address, + timeout, + ) + except PicoTLSException: + issuer_certificate = None else: issuer_certificate = None diff --git a/src/niquests/extensions/_picotls.py b/src/niquests/extensions/_picotls.py index d0e59af49a..99a5a0cafc 100644 --- a/src/niquests/extensions/_picotls.py +++ b/src/niquests/extensions/_picotls.py @@ -284,6 +284,10 @@ ] +class PicoTLSException(Exception): + pass + + def bytes_to_num(b): return int.from_bytes(b, "big") @@ -438,7 +442,9 @@ def handle_server_hello(server_hello): handshake_type = server_hello[0] SERVER_HELLO = 0x2 - assert handshake_type == SERVER_HELLO + + if handshake_type != SERVER_HELLO: + raise PicoTLSException # server_hello_len = server_hello[1:4] # server_version = server_hello[4:6] @@ -449,7 +455,8 @@ def handle_server_hello(server_hello): session_id = server_hello[39 : 39 + session_id_len] cipher_suite = server_hello[39 + session_id_len : 39 + session_id_len + 2] - assert cipher_suite == TLS_AES_128_GCM_SHA256 + if cipher_suite != TLS_AES_128_GCM_SHA256: + raise PicoTLSException # compression_method = server_hello[39 + session_id_len + 2 : 39 + session_id_len + 3] @@ -471,7 +478,10 @@ def handle_server_hello(server_hello): continue group = extensions[ptr + 4 : ptr + 6] SECP256R1_GROUP = b"\x00\x17" - assert group == SECP256R1_GROUP + + if group != SECP256R1_GROUP: + raise PicoTLSException + key_exchange_len = bytes_to_num(extensions[ptr + 6 : ptr + 8]) public_ec_key = extensions[ptr + 8 : ptr + 8 + key_exchange_len] @@ -658,7 +668,8 @@ def handle_server_cert(server_cert_data): handshake_type = server_cert_data[0] CERTIFICATE = 0x0B - assert handshake_type == CERTIFICATE + if handshake_type != CERTIFICATE: + raise PicoTLSException # certificate_payload_len = bytes_to_num(server_cert_data[1:4]) certificate_list_len = bytes_to_num(server_cert_data[5:8]) @@ -679,16 +690,19 @@ def handle_server_cert(server_cert_data): def handle_encrypted_extensions(msg): ENCRYPTED_EXTENSIONS = 0x8 - assert msg[0] == ENCRYPTED_EXTENSIONS + if msg[0] != ENCRYPTED_EXTENSIONS: + raise PicoTLSException extensions_length = bytes_to_num(msg[1:4]) - assert len(msg[4:]) >= extensions_length + if len(msg[4:]) < extensions_length: + raise PicoTLSException return msg[4 + extensions_length :] # ignore the rest def recv_tls_and_decrypt(s, key, nonce, seq_num): rec_type, encrypted_msg = recv_tls(s) - assert rec_type == APPLICATION_DATA + if rec_type != APPLICATION_DATA: + raise PicoTLSException msg_type, msg = do_authenticated_decryption( key, nonce, seq_num, APPLICATION_DATA, encrypted_msg @@ -698,7 +712,8 @@ def recv_tls_and_decrypt(s, key, nonce, seq_num): async def async_recv_tls_and_decrypt(s, key, nonce, seq_num): rec_type, encrypted_msg = await async_recv_tls(s) - assert rec_type == APPLICATION_DATA + if rec_type != APPLICATION_DATA: + raise PicoTLSException msg_type, msg = do_authenticated_decryption( key, nonce, seq_num, APPLICATION_DATA, encrypted_msg @@ -730,7 +745,8 @@ def recv_tls(s): rec_type = recv_num_bytes(s, 1) tls_version = recv_num_bytes(s, 2) - assert tls_version == LEGACY_TLS_VERSION + if tls_version != LEGACY_TLS_VERSION: + raise PicoTLSException rec_len = bytes_to_num(recv_num_bytes(s, 2)) rec = recv_num_bytes(s, rec_len) @@ -746,7 +762,8 @@ async def async_recv_tls(s): rec_type = await async_recv_num_bytes(s, 1) tls_version = await async_recv_num_bytes(s, 2) - assert tls_version == LEGACY_TLS_VERSION + if tls_version != LEGACY_TLS_VERSION: + raise PicoTLSException rec_len = bytes_to_num(await async_recv_num_bytes(s, 2)) rec = await async_recv_num_bytes(s, rec_len) From 5f7b3daa8b2edc6df515101759a35ab0a63130a7 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Mon, 21 Oct 2024 09:48:45 +0200 Subject: [PATCH 2/4] :sparkle: automated keep-alive based on pings for http2+ --- .pre-commit-config.yaml | 2 +- README.md | 1 + pyproject.toml | 2 +- src/niquests/_async.py | 13 +++++++++++++ src/niquests/adapters.py | 22 +++++++++++++++++++++- src/niquests/sessions.py | 19 +++++++++++++++++++ 6 files changed, 56 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 764173bbe7..8227d5d15b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,4 +28,4 @@ repos: - id: mypy args: [--check-untyped-defs] exclude: 'tests/|noxfile.py' - additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.10.903', 'wassima>=1.0.1', 'idna', 'kiss_headers'] + additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.11.900', 'wassima>=1.0.1', 'idna', 'kiss_headers'] diff --git a/README.md b/README.md index 1ad5f1e073..b2ab911c6a 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Niquests, is the “**Safest**, **Fastest[^10]**, **Easiest**, and **Most advanc | `Early Responses` | ✅ | ❌ | ❌ | ❌ | | `WebSocket over HTTP/1` | ✅ | ❌[^14] | ❌[^14] | ✅ | | `WebSocket over HTTP/2 and HTTP/3` | ✅[^13] | ❌ | ❌ | ❌ | +| `Automatic Ping for HTTP/2+` | ✅ | N/A | ❌ | N/A |
diff --git a/pyproject.toml b/pyproject.toml index 339609fcaf..1bc5b544e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dynamic = ["version"] dependencies = [ "charset_normalizer>=2,<4", "idna>=2.5,<4", - "urllib3.future>=2.10.904,<3", + "urllib3.future>=2.11.900,<3", "wassima>=1.0.1,<2", "kiss_headers>=2,<4", ] diff --git a/src/niquests/_async.py b/src/niquests/_async.py index a25ffca34e..4af57cf567 100644 --- a/src/niquests/_async.py +++ b/src/niquests/_async.py @@ -130,6 +130,8 @@ def __init__( pool_connections: int = DEFAULT_POOLSIZE, pool_maxsize: int = DEFAULT_POOLSIZE, happy_eyeballs: bool | int = False, + keepalive_delay: float | int | None = 300.0, + keepalive_idle_window: float | int | None = 60.0, ): if [disable_ipv4, disable_ipv6].count(True) == 2: raise RuntimeError("Cannot disable both IPv4 and IPv6") @@ -195,6 +197,9 @@ def __init__( self._happy_eyeballs = happy_eyeballs + self._keepalive_delay = keepalive_delay + self._keepalive_idle_window = keepalive_idle_window + #: SSL Verification default. #: Defaults to `True`, requiring requests to verify the TLS certificate at the #: remote end. @@ -253,6 +258,8 @@ def __init__( pool_connections=pool_connections, pool_maxsize=pool_maxsize, happy_eyeballs=happy_eyeballs, + keepalive_delay=keepalive_delay, + keepalive_idle_window=keepalive_idle_window, ), ) self.mount( @@ -269,6 +276,8 @@ def __init__( pool_connections=pool_connections, pool_maxsize=pool_maxsize, happy_eyeballs=happy_eyeballs, + keepalive_delay=keepalive_delay, + keepalive_idle_window=keepalive_idle_window, ), ) @@ -436,6 +445,8 @@ async def on_early_response(early_response: Response) -> None: pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, happy_eyeballs=self._happy_eyeballs, + keepalive_delay=self._keepalive_delay, + keepalive_idle_window=self._keepalive_idle_window, ), ) self.mount( @@ -452,6 +463,8 @@ async def on_early_response(early_response: Response) -> None: pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, happy_eyeballs=self._happy_eyeballs, + keepalive_delay=self._keepalive_delay, + keepalive_idle_window=self._keepalive_idle_window, ), ) diff --git a/src/niquests/adapters.py b/src/niquests/adapters.py index 42c0c3249c..4e6a8d574c 100644 --- a/src/niquests/adapters.py +++ b/src/niquests/adapters.py @@ -333,6 +333,8 @@ class HTTPAdapter(BaseAdapter): "_disable_ipv4", "_disable_ipv6", "_happy_eyeballs", + "_keepalive_delay", + "_keepalive_idle_window", ] def __init__( @@ -341,7 +343,7 @@ def __init__( pool_maxsize: int = DEFAULT_POOLSIZE, max_retries: RetryType = DEFAULT_RETRIES, pool_block: bool = DEFAULT_POOLBLOCK, - *, # todo: revert if any complaint about it... :s + *, quic_cache_layer: CacheLayerAltSvcType | None = None, disable_http1: bool = False, disable_http2: bool = False, @@ -352,6 +354,8 @@ def __init__( disable_ipv4: bool = False, disable_ipv6: bool = False, happy_eyeballs: bool | int = False, + keepalive_delay: float | int | None = 300.0, + keepalive_idle_window: float | int | None = 60.0, ): if isinstance(max_retries, bool): self.max_retries: RetryType = False @@ -383,6 +387,8 @@ def __init__( self._disable_ipv4 = disable_ipv4 self._disable_ipv6 = disable_ipv6 self._happy_eyeballs = happy_eyeballs + self._keepalive_delay = keepalive_delay + self._keepalive_idle_window = keepalive_idle_window #: we keep a list of pending (lazy) response self._promises: dict[str, Response] = {} @@ -413,6 +419,8 @@ def __init__( source_address=source_address, socket_family=resolve_socket_family(disable_ipv4, disable_ipv6), happy_eyeballs=happy_eyeballs, + keepalive_delay=keepalive_delay, + keepalive_idle_window=keepalive_idle_window, ) def __getstate__(self) -> dict[str, typing.Any | None]: @@ -447,6 +455,8 @@ def __setstate__(self, state): source_address=self._source_address, socket_family=resolve_socket_family(self._disable_ipv4, self._disable_ipv6), happy_eyeballs=self._happy_eyeballs, + keepalive_delay=self._keepalive_delay, + keepalive_idle_window=self._keepalive_idle_window, ) def init_poolmanager( @@ -1335,6 +1345,8 @@ class AsyncHTTPAdapter(AsyncBaseAdapter): "_disable_ipv4", "_disable_ipv6", "_happy_eyeballs", + "_keepalive_delay", + "_keepalive_idle_window", ] def __init__( @@ -1354,6 +1366,8 @@ def __init__( disable_ipv4: bool = False, disable_ipv6: bool = False, happy_eyeballs: bool | int = False, + keepalive_delay: float | int | None = 300.0, + keepalive_idle_window: float | int | None = 60.0, ): if isinstance(max_retries, bool): self.max_retries: RetryType = False @@ -1386,6 +1400,8 @@ def __init__( self._disable_ipv4 = disable_ipv4 self._disable_ipv6 = disable_ipv6 self._happy_eyeballs = happy_eyeballs + self._keepalive_delay = keepalive_delay + self._keepalive_idle_window = keepalive_idle_window #: we keep a list of pending (lazy) response self._promises: dict[str, Response | AsyncResponse] = {} @@ -1415,6 +1431,8 @@ def __init__( source_address=source_address, socket_family=resolve_socket_family(disable_ipv4, disable_ipv6), happy_eyeballs=happy_eyeballs, + keepalive_delay=keepalive_delay, + keepalive_idle_window=keepalive_idle_window, ) def __getstate__(self) -> dict[str, typing.Any | None]: @@ -1449,6 +1467,8 @@ def __setstate__(self, state): source_address=self._source_address, socket_family=resolve_socket_family(self._disable_ipv4, self._disable_ipv6), happy_eyeballs=self._happy_eyeballs, + keepalive_delay=self._keepalive_delay, + keepalive_idle_window=self._keepalive_idle_window, ) def init_poolmanager( diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index 02772b5f68..0186e0df18 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -230,6 +230,8 @@ class Session: "_pool_connections", "_pool_maxsize", "_happy_eyeballs", + "_keepalive_delay", + "_keepalive_idle_window", ] def __init__( @@ -248,6 +250,8 @@ def __init__( pool_connections: int = DEFAULT_POOLSIZE, pool_maxsize: int = DEFAULT_POOLSIZE, happy_eyeballs: bool | int = False, + keepalive_delay: float | int | None = 300.0, + keepalive_idle_window: float | int | None = 60.0, ): """ :param resolver: Specify a DNS resolver that should be used within this Session. @@ -324,6 +328,9 @@ def __init__( self._happy_eyeballs = happy_eyeballs + self._keepalive_delay = keepalive_delay + self._keepalive_idle_window = keepalive_idle_window + #: SSL Verification default. #: Defaults to `True`, requiring requests to verify the TLS certificate at the #: remote end. @@ -380,6 +387,8 @@ def __init__( pool_connections=pool_connections, pool_maxsize=pool_maxsize, happy_eyeballs=happy_eyeballs, + keepalive_delay=keepalive_delay, + keepalive_idle_window=keepalive_idle_window, ), ) self.mount( @@ -395,6 +404,8 @@ def __init__( pool_connections=pool_connections, pool_maxsize=pool_maxsize, happy_eyeballs=happy_eyeballs, + keepalive_delay=keepalive_delay, + keepalive_idle_window=keepalive_idle_window, ), ) @@ -1178,6 +1189,8 @@ def on_early_response(early_response) -> None: pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, happy_eyeballs=self._happy_eyeballs, + keepalive_delay=self._keepalive_delay, + keepalive_idle_window=self._keepalive_idle_window, ), ) self.mount( @@ -1194,6 +1207,8 @@ def on_early_response(early_response) -> None: pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, happy_eyeballs=self._happy_eyeballs, + keepalive_delay=self._keepalive_delay, + keepalive_idle_window=self._keepalive_idle_window, ), ) @@ -1441,6 +1456,8 @@ def __setstate__(self, state): pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, happy_eyeballs=self._happy_eyeballs, + keepalive_delay=self._keepalive_delay, + keepalive_idle_window=self._keepalive_idle_window, ), ) self.mount( @@ -1454,6 +1471,8 @@ def __setstate__(self, state): pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, happy_eyeballs=self._happy_eyeballs, + keepalive_delay=self._keepalive_delay, + keepalive_idle_window=self._keepalive_idle_window, ), ) From 3f33d246e90dc598828cfe4c0b592ec6bf119523 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Mon, 21 Oct 2024 09:48:57 +0200 Subject: [PATCH 3/4] :bookmark: bump to 3.10 --- HISTORY.md | 11 +++++++++++ src/niquests/__version__.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index b6e8c1682b..07d85445a9 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,17 @@ Release History =============== +3.10.0 (2024-10-21) +------------------ + +**Added** +- Automatic Advanced Keep-Alive for HTTP/2 and HTTP/3 over QUIC by sending PING frames. + New Session, and Adapter parameters are now available: `keepalive_delay`, and `keepalive_idle_window`. + This greatly improves your daily experience working with HTTP/2+ remote peers. + +**Fixed** +- Unshielded picotls assertion error in Python < 3.10 when trying to fetch the peer intermediate certificate. + 3.9.1 (2024-10-13) ------------------ diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index 399e2182f9..68008f3a72 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.9.1" +__version__ = "3.10.0" -__build__: int = 0x030901 +__build__: int = 0x031000 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" From 0d9b7aaaae9f044f1c12d9856235ff5a8ef83781 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Mon, 21 Oct 2024 09:49:16 +0200 Subject: [PATCH 4/4] :pencil: add note about keep alive for http2+ --- docs/user/quickstart.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index bb488d9ef3..576f3ec53e 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -1249,6 +1249,32 @@ See:: .. note:: The given example are really basic ones. You may adjust at will the settings and algorithm to match your requisites. +Keep-Alive +---------- + +.. note:: Available since Niquests v3.10 and before this only HTTP/1.1 were kept alive properly. + +Niquests can automatically make sure that your HTTP connection is kept alive +no matter the used protocol using a discrete scheduled task for each host. + +.. code-block:: python + + import niquests + + sess = niquests.Session(keepalive_delay=300, keepalive_idle_window=60) # already the defaults!, you don't need to specify anything + +In that example, we indicate that we wish to keep a connection alive for 5 minutes and +eventually send ping every 60s after the connection was idle. (Those values are the default ones!) + +The pings are only sent when using HTTP/2 or HTTP/3 over QUIC. Any connection activity is considered as used, therefor +making the ping only 60s after zero activity. If the connection receive unsolicited data, it is also considered used. + +.. note:: Setting either keepalive_delay or keepalive_idle_window to None disable this feature. + +.. warning:: We do not recommend setting anything lower than 30s for keepalive_idle_window. Anything lower than 1s is considered to be 1s. High frequency ping will lower the performance of your connection pool. And probably end up by getting kicked out by the server. + +Once the ``keepalive_delay`` passed, we do not close the connection, we simply cease to ensure it is alive. This is purely for backward compatibility with our predecessor, as some host may retain the connection for hours. + ----------------------- Ready for more? Check out the :ref:`advanced ` section.