From 492e10c21dcf33bd3d16f1c6c17aed052fc37832 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 11 Aug 2020 21:50:01 +0100 Subject: [PATCH 01/10] Fix max_keepalive_connections config --- httpcore/_async/connection_pool.py | 6 +++--- httpcore/_sync/connection_pool.py | 6 +++--- tests/async_tests/test_interfaces.py | 3 +-- tests/conftest.py | 8 ++++---- tests/sync_tests/test_interfaces.py | 3 +-- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/httpcore/_async/connection_pool.py b/httpcore/_async/connection_pool.py index ce2bb439..67025c80 100644 --- a/httpcore/_async/connection_pool.py +++ b/httpcore/_async/connection_pool.py @@ -105,7 +105,7 @@ def __init__( self._ssl_context = SSLContext() if ssl_context is None else ssl_context self._max_connections = max_connections - self._max_keepalive = max_keepalive + self._max_keepalive_connections = max_keepalive_connections self._keepalive_expiry = keepalive_expiry self._http2 = http2 self._uds = uds @@ -259,8 +259,8 @@ async def _response_closed(self, connection: AsyncHTTPConnection) -> None: elif connection.state == ConnectionState.IDLE: num_connections = len(self._get_all_connections()) if ( - self._max_keepalive is not None - and num_connections > self._max_keepalive + self._max_keepalive_connections is not None + and num_connections > self._max_keepalive_connections ): remove_from_pool = True close_connection = True diff --git a/httpcore/_sync/connection_pool.py b/httpcore/_sync/connection_pool.py index 0270fd46..812e2ec6 100644 --- a/httpcore/_sync/connection_pool.py +++ b/httpcore/_sync/connection_pool.py @@ -105,7 +105,7 @@ def __init__( self._ssl_context = SSLContext() if ssl_context is None else ssl_context self._max_connections = max_connections - self._max_keepalive = max_keepalive + self._max_keepalive_connections = max_keepalive_connections self._keepalive_expiry = keepalive_expiry self._http2 = http2 self._uds = uds @@ -259,8 +259,8 @@ def _response_closed(self, connection: SyncHTTPConnection) -> None: elif connection.state == ConnectionState.IDLE: num_connections = len(self._get_all_connections()) if ( - self._max_keepalive is not None - and num_connections > self._max_keepalive + self._max_keepalive_connections is not None + and num_connections > self._max_keepalive_connections ): remove_from_pool = True close_connection = True diff --git a/tests/async_tests/test_interfaces.py b/tests/async_tests/test_interfaces.py index 94635b3c..33cebde6 100644 --- a/tests/async_tests/test_interfaces.py +++ b/tests/async_tests/test_interfaces.py @@ -1,6 +1,5 @@ -import ssl import platform -from pathlib import Path +import ssl import pytest diff --git a/tests/conftest.py b/tests/conftest.py index 9a64c425..32f87635 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,14 @@ import asyncio +import contextlib +import os import ssl import threading -import typing -import contextlib import time +import typing -import os -import uvicorn import pytest import trustme +import uvicorn from mitmproxy import options, proxy from mitmproxy.tools.dump import DumpMaster diff --git a/tests/sync_tests/test_interfaces.py b/tests/sync_tests/test_interfaces.py index c000a644..303138fd 100644 --- a/tests/sync_tests/test_interfaces.py +++ b/tests/sync_tests/test_interfaces.py @@ -1,6 +1,5 @@ -import ssl import platform -from pathlib import Path +import ssl import pytest From 3062506d65de9ea82c6f4077ef2d75da2ca999d8 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 21 Sep 2020 16:12:53 +0100 Subject: [PATCH 02/10] Use .arequest for the async version of the API --- README.md | 20 +++++++++++++++- docs/api.md | 2 +- docs/index.md | 22 ++++++++++++++++-- httpcore/_async/base.py | 2 +- httpcore/_async/connection.py | 6 ++--- httpcore/_async/connection_pool.py | 6 ++--- httpcore/_async/http11.py | 2 +- httpcore/_async/http2.py | 6 ++--- httpcore/_async/http_proxy.py | 8 +++---- tests/async_tests/test_interfaces.py | 34 ++++++++++++++-------------- unasync.py | 1 + 11 files changed, 73 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 4e77dd1c..c8daf613 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,27 @@ $ pip install httpcore[http2] Here's an example of making an HTTP GET request using `httpcore`... +```python +with httpcore.SyncConnectionPool() as http: + http_version, status_code, reason_phrase, headers, stream = http.request( + method=b'GET', + url=(b'https', b'example.org', 443, b'/'), + headers=[(b'host', b'example.org'), (b'user-agent', 'httpcore')] + ) + + try: + body = b''.join([chunk for chunk in stream]) + finally: + stream.close() + + print(status_code, body) +``` + +Or, using async... + ```python async with httpcore.AsyncConnectionPool() as http: - http_version, status_code, reason_phrase, headers, stream = await http.request( + http_version, status_code, reason_phrase, headers, stream = await http.arequest( method=b'GET', url=(b'https', b'example.org', 443, b'/'), headers=[(b'host', b'example.org'), (b'user-agent', 'httpcore')] diff --git a/docs/api.md b/docs/api.md index dfda0013..3bbde423 100644 --- a/docs/api.md +++ b/docs/api.md @@ -7,7 +7,7 @@ interface which transport classes need to implement. ::: httpcore.AsyncHTTPTransport :docstring: - :members: request aclose + :members: arequest aclose ::: httpcore.AsyncByteStream :docstring: diff --git a/docs/index.md b/docs/index.md index df930dc0..c8daf613 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,11 +41,29 @@ $ pip install httpcore[http2] Here's an example of making an HTTP GET request using `httpcore`... +```python +with httpcore.SyncConnectionPool() as http: + http_version, status_code, reason_phrase, headers, stream = http.request( + method=b'GET', + url=(b'https', b'example.org', 443, b'/'), + headers=[(b'host', b'example.org'), (b'user-agent', 'httpcore')] + ) + + try: + body = b''.join([chunk for chunk in stream]) + finally: + stream.close() + + print(status_code, body) +``` + +Or, using async... + ```python async with httpcore.AsyncConnectionPool() as http: - http_version, status_code, reason_phrase, headers, stream = await http.request( + http_version, status_code, reason_phrase, headers, stream = await http.arequest( method=b'GET', - url=(b'https', b'example.org', 433, b'/'), + url=(b'https', b'example.org', 443, b'/'), headers=[(b'host', b'example.org'), (b'user-agent', 'httpcore')] ) diff --git a/httpcore/_async/base.py b/httpcore/_async/base.py index 9fdea57c..e0a50d12 100644 --- a/httpcore/_async/base.py +++ b/httpcore/_async/base.py @@ -61,7 +61,7 @@ class AsyncHTTPTransport: the `request` method, and optionally the `close` method. """ - async def request( + async def arequest( self, method: bytes, url: URL, diff --git a/httpcore/_async/connection.py b/httpcore/_async/connection.py index 95ddb526..a2cd1f3e 100644 --- a/httpcore/_async/connection.py +++ b/httpcore/_async/connection.py @@ -65,7 +65,7 @@ def request_lock(self) -> AsyncLock: self._request_lock = self.backend.create_lock() return self._request_lock - async def request( + async def arequest( self, method: bytes, url: URL, @@ -91,9 +91,9 @@ async def request( assert self.connection is not None logger.trace( - "connection.request method=%r url=%r headers=%r", method, url, headers + "connection.arequest method=%r url=%r headers=%r", method, url, headers ) - return await self.connection.request(method, url, headers, stream, timeout) + return await self.connection.arequest(method, url, headers, stream, timeout) async def _open_socket(self, timeout: TimeoutDict = None) -> AsyncSocketStream: scheme, hostname, port = self.origin diff --git a/httpcore/_async/connection_pool.py b/httpcore/_async/connection_pool.py index 67025c80..5ae81d25 100644 --- a/httpcore/_async/connection_pool.py +++ b/httpcore/_async/connection_pool.py @@ -37,7 +37,7 @@ def __init__( callback: Callable, ) -> None: """ - A wrapper around the response stream that we return from `.request()`. + A wrapper around the response stream that we return from `.arequest()`. Ensures that when `stream.aclose()` is called, the connection pool is notified via a callback. @@ -144,7 +144,7 @@ def _connection_acquiry_lock(self) -> AsyncLock: self._internal_connection_acquiry_lock = self._backend.create_lock() return self._internal_connection_acquiry_lock - async def request( + async def arequest( self, method: bytes, url: URL, @@ -185,7 +185,7 @@ async def request( logger.trace("reuse connection=%r", connection) try: - response = await connection.request( + response = await connection.arequest( method, url, headers=headers, stream=stream, timeout=timeout ) except NewConnectionRequired: diff --git a/httpcore/_async/http11.py b/httpcore/_async/http11.py index e8d3c77c..4218de4b 100644 --- a/httpcore/_async/http11.py +++ b/httpcore/_async/http11.py @@ -49,7 +49,7 @@ def mark_as_ready(self) -> None: if self.state == ConnectionState.IDLE: self.state = ConnectionState.READY - async def request( + async def arequest( self, method: bytes, url: URL, diff --git a/httpcore/_async/http2.py b/httpcore/_async/http2.py index 460158e4..e1643942 100644 --- a/httpcore/_async/http2.py +++ b/httpcore/_async/http2.py @@ -93,7 +93,7 @@ def mark_as_ready(self) -> None: if self.state == ConnectionState.IDLE: self.state = ConnectionState.READY - async def request( + async def arequest( self, method: bytes, url: URL, @@ -123,7 +123,7 @@ async def request( h2_stream = AsyncHTTP2Stream(stream_id=stream_id, connection=self) self.streams[stream_id] = h2_stream self.events[stream_id] = [] - return await h2_stream.request(method, url, headers, stream, timeout) + return await h2_stream.arequest(method, url, headers, stream, timeout) except Exception: self.max_streams_semaphore.release() raise @@ -274,7 +274,7 @@ def __init__(self, stream_id: int, connection: AsyncHTTP2Connection) -> None: self.stream_id = stream_id self.connection = connection - async def request( + async def arequest( self, method: bytes, url: URL, diff --git a/httpcore/_async/http_proxy.py b/httpcore/_async/http_proxy.py index a93bb282..8b23e48c 100644 --- a/httpcore/_async/http_proxy.py +++ b/httpcore/_async/http_proxy.py @@ -76,7 +76,7 @@ def __init__( max_keepalive=max_keepalive, ) - async def request( + async def arequest( self, method: bytes, url: URL, @@ -155,7 +155,7 @@ async def _forward_request( reason_phrase, headers, stream, - ) = await connection.request( + ) = await connection.arequest( method, url, headers=headers, stream=stream, timeout=timeout ) @@ -207,7 +207,7 @@ async def _tunnel_request( proxy_reason_phrase, _, proxy_stream, - ) = await proxy_connection.request( + ) = await proxy_connection.arequest( b"CONNECT", connect_url, headers=connect_headers, timeout=timeout ) logger.trace( @@ -249,7 +249,7 @@ async def _tunnel_request( reason_phrase, headers, stream, - ) = await connection.request( + ) = await connection.arequest( method, url, headers=headers, stream=stream, timeout=timeout, ) diff --git a/tests/async_tests/test_interfaces.py b/tests/async_tests/test_interfaces.py index 33cebde6..02fd3d50 100644 --- a/tests/async_tests/test_interfaces.py +++ b/tests/async_tests/test_interfaces.py @@ -23,7 +23,7 @@ async def test_http_request() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -40,7 +40,7 @@ async def test_https_request() -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -58,7 +58,7 @@ async def test_request_unsupported_protocol() -> None: url = (b"ftp", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] with pytest.raises(httpcore.UnsupportedProtocol): - await http.request(method, url, headers) + await http.arequest(method, url, headers) @pytest.mark.usefixtures("async_environment") @@ -67,7 +67,7 @@ async def test_http2_request() -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -84,7 +84,7 @@ async def test_closing_http_request() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org"), (b"connection", b"close")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -101,7 +101,7 @@ async def test_http_request_reuse_connection() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -114,7 +114,7 @@ async def test_http_request_reuse_connection() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -131,7 +131,7 @@ async def test_https_request_reuse_connection() -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -144,7 +144,7 @@ async def test_https_request_reuse_connection() -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -161,7 +161,7 @@ async def test_http_request_cannot_reuse_dropped_connection() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -178,7 +178,7 @@ async def test_http_request_cannot_reuse_dropped_connection() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -199,7 +199,7 @@ async def test_http_proxy(proxy_server: URL, proxy_mode: str) -> None: async with httpcore.AsyncHTTPProxy( proxy_server, proxy_mode=proxy_mode, max_connections=max_connections, ) as http: - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -216,7 +216,7 @@ async def test_http_request_local_address() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) body = await read_body(stream) @@ -245,7 +245,7 @@ async def test_proxy_https_requests( max_connections=max_connections, http2=http2, ) as http: - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) _ = await read_body(stream) @@ -290,8 +290,8 @@ async def test_connection_pool_get_connection_info( url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - _, _, _, _, stream_1 = await http.request(method, url, headers) - _, _, _, _, stream_2 = await http.request(method, url, headers) + _, _, _, _, stream_1 = await http.arequest(method, url, headers) + _, _, _, _, stream_2 = await http.arequest(method, url, headers) try: stats = await http.get_connection_info() @@ -319,7 +319,7 @@ async def test_http_request_unix_domain_socket(uds_server) -> None: method = b"GET" url = (b"http", b"localhost", None, b"/") headers = [(b"host", b"localhost")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) assert http_version == b"HTTP/1.1" diff --git a/unasync.py b/unasync.py index 1c74fcb4..ecf5c39b 100755 --- a/unasync.py +++ b/unasync.py @@ -12,6 +12,7 @@ ('async with', 'with'), ('async for', 'for'), ('await ', ''), + ('arequest', 'request'), ('aclose', 'close'), ('aclose_func', 'close_func'), ('aiterator', 'iterator'), From e3b9ec4c67e881a1bcf4b0f387652efe76a39ff1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 21 Sep 2020 16:21:17 +0100 Subject: [PATCH 03/10] Linting --- httpcore/_async/http_proxy.py | 6 +++++- httpcore/_backends/anyio.py | 5 ++++- httpcore/_backends/curio.py | 6 ++++-- httpcore/_sync/http_proxy.py | 6 +++++- tests/async_tests/test_interfaces.py | 7 ++++++- tests/sync_tests/test_interfaces.py | 7 ++++++- 6 files changed, 30 insertions(+), 7 deletions(-) diff --git a/httpcore/_async/http_proxy.py b/httpcore/_async/http_proxy.py index 2cf127de..dc3fc9d6 100644 --- a/httpcore/_async/http_proxy.py +++ b/httpcore/_async/http_proxy.py @@ -250,7 +250,11 @@ async def _tunnel_request( headers, stream, ) = await connection.arequest( - method, url, headers=headers, stream=stream, timeout=timeout, + method, + url, + headers=headers, + stream=stream, + timeout=timeout, ) wrapped_stream = ResponseByteStream( diff --git a/httpcore/_backends/anyio.py b/httpcore/_backends/anyio.py index c1f773a0..0921be0f 100644 --- a/httpcore/_backends/anyio.py +++ b/httpcore/_backends/anyio.py @@ -31,7 +31,10 @@ def get_http_version(self) -> str: return "HTTP/2" if alpn_protocol == "h2" else "HTTP/1.1" async def start_tls( - self, hostname: bytes, ssl_context: SSLContext, timeout: TimeoutDict, + self, + hostname: bytes, + ssl_context: SSLContext, + timeout: TimeoutDict, ) -> "SocketStream": connect_timeout = timeout.get("connect") try: diff --git a/httpcore/_backends/curio.py b/httpcore/_backends/curio.py index 8064984d..9f69850b 100644 --- a/httpcore/_backends/curio.py +++ b/httpcore/_backends/curio.py @@ -98,7 +98,8 @@ async def start_tls( ) await curio.timeout_after( - connect_timeout, wrapped_sock.do_handshake(), + connect_timeout, + wrapped_sock.do_handshake(), ) return SocketStream(wrapped_sock) @@ -164,7 +165,8 @@ async def open_tcp_stream( with map_exceptions(exc_map): sock: curio.io.Socket = await curio.timeout_after( - connect_timeout, curio.open_connection(hostname, port, **kwargs), + connect_timeout, + curio.open_connection(hostname, port, **kwargs), ) return SocketStream(sock) diff --git a/httpcore/_sync/http_proxy.py b/httpcore/_sync/http_proxy.py index 12dabed7..71805140 100644 --- a/httpcore/_sync/http_proxy.py +++ b/httpcore/_sync/http_proxy.py @@ -250,7 +250,11 @@ def _tunnel_request( headers, stream, ) = connection.request( - method, url, headers=headers, stream=stream, timeout=timeout, + method, + url, + headers=headers, + stream=stream, + timeout=timeout, ) wrapped_stream = ResponseByteStream( diff --git a/tests/async_tests/test_interfaces.py b/tests/async_tests/test_interfaces.py index cd6561a9..91269cf7 100644 --- a/tests/async_tests/test_interfaces.py +++ b/tests/async_tests/test_interfaces.py @@ -290,7 +290,12 @@ async def test_proxy_https_requests( {"https://example.org": ["HTTP/1.1, ACTIVE", "HTTP/1.1, ACTIVE"]}, {}, ), - (True, 0.0, {"https://example.org": ["HTTP/2, ACTIVE, 2 streams"]}, {},), + ( + True, + 0.0, + {"https://example.org": ["HTTP/2, ACTIVE, 2 streams"]}, + {}, + ), ], ) @pytest.mark.anyio diff --git a/tests/sync_tests/test_interfaces.py b/tests/sync_tests/test_interfaces.py index e7613a0e..681b8742 100644 --- a/tests/sync_tests/test_interfaces.py +++ b/tests/sync_tests/test_interfaces.py @@ -290,7 +290,12 @@ def test_proxy_https_requests( {"https://example.org": ["HTTP/1.1, ACTIVE", "HTTP/1.1, ACTIVE"]}, {}, ), - (True, 0.0, {"https://example.org": ["HTTP/2, ACTIVE, 2 streams"]}, {},), + ( + True, + 0.0, + {"https://example.org": ["HTTP/2, ACTIVE, 2 streams"]}, + {}, + ), ], ) From 63dd4ec0c9bb57fe6815d1caddfa91b0b30ca703 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 21 Sep 2020 16:25:36 +0100 Subject: [PATCH 04/10] Update tests --- tests/async_tests/test_interfaces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/async_tests/test_interfaces.py b/tests/async_tests/test_interfaces.py index 91269cf7..a6f0f687 100644 --- a/tests/async_tests/test_interfaces.py +++ b/tests/async_tests/test_interfaces.py @@ -369,7 +369,7 @@ async def test_max_keepalive_connections_handled_correctly( connections_streams = [] for _ in range(connections_number): - _, _, _, _, stream = await http.request(method, url, headers) + _, _, _, _, stream = await http.arequest(method, url, headers) connections_streams.append(stream) try: @@ -388,7 +388,7 @@ async def test_explicit_backend_name() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.request( + http_version, status_code, reason, headers, stream = await http.arequest( method, url, headers ) await read_body(stream) From 703ad1a2a55bf04df699a36171b3f72d4586cdb3 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 21 Sep 2020 17:14:17 +0100 Subject: [PATCH 05/10] Switch to transport API with 'ext' interface --- httpcore/_async/base.py | 20 ++--- httpcore/_async/connection.py | 11 ++- httpcore/_async/connection_pool.py | 19 ++--- httpcore/_async/http11.py | 12 +-- httpcore/_async/http2.py | 32 ++++---- httpcore/_async/http_proxy.py | 74 ++++++++--------- httpcore/_sync/base.py | 20 ++--- httpcore/_sync/connection.py | 11 ++- httpcore/_sync/connection_pool.py | 19 ++--- httpcore/_sync/http11.py | 12 +-- httpcore/_sync/http2.py | 32 ++++---- httpcore/_sync/http_proxy.py | 74 ++++++++--------- tests/async_tests/test_interfaces.py | 116 +++++++++------------------ tests/sync_tests/test_interfaces.py | 116 +++++++++------------------ 14 files changed, 236 insertions(+), 332 deletions(-) diff --git a/httpcore/_async/base.py b/httpcore/_async/base.py index 9b0fc463..cf449f42 100644 --- a/httpcore/_async/base.py +++ b/httpcore/_async/base.py @@ -1,8 +1,8 @@ import enum from types import TracebackType -from typing import AsyncIterator, List, Tuple, Type +from typing import AsyncIterator, Tuple, Type -from .._types import URL, Headers, T, TimeoutDict +from .._types import URL, Headers, T class NewConnectionRequired(Exception): @@ -67,8 +67,8 @@ async def arequest( url: URL, headers: Headers = None, stream: AsyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, AsyncByteStream, dict]: """ The interface for sending a single HTTP request, and returning a response. @@ -80,23 +80,17 @@ async def arequest( * **headers** - `Optional[List[Tuple[bytes, bytes]]]` - Any HTTP headers to send with the request. * **stream** - `Optional[AsyncByteStream]` - The body of the HTTP request. - * **timeout** - `Optional[Dict[str, Optional[float]]]` - A dictionary of - timeout values for I/O operations. Supported keys are "pool" for acquiring a - connection from the connection pool, "read" for reading from the connection, - "write" for writing to the connection and "connect" for opening the connection. - Values are floating point seconds. + * **ext** - `Optional[dict]` - A dictionary of optional extensions. ** Returns:** - A five-tuple of: + A four-tuple of: - * **http_version** - `bytes` - The HTTP version used by the server, - such as `b'HTTP/1.1'`. * **status_code** - `int` - The HTTP status code, such as `200`. - * **reason_phrase** - `bytes` - Any HTTP reason phrase, such as `b'OK'`. * **headers** - `List[Tuple[bytes, bytes]]` - Any HTTP headers included on the response. * **stream** - `AsyncByteStream` - The body of the HTTP response. + * **ext** - `dict` - A dictionary of optional extensions. """ raise NotImplementedError() # pragma: nocover diff --git a/httpcore/_async/connection.py b/httpcore/_async/connection.py index 35fe2a26..258d20d5 100644 --- a/httpcore/_async/connection.py +++ b/httpcore/_async/connection.py @@ -1,5 +1,5 @@ from ssl import SSLContext -from typing import List, Optional, Tuple +from typing import Optional, Tuple, cast from .._backends.auto import AsyncBackend, AsyncLock, AsyncSocketStream, AutoBackend from .._types import URL, Headers, Origin, TimeoutDict @@ -72,9 +72,12 @@ async def arequest( url: URL, headers: Headers = None, stream: AsyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, AsyncByteStream, dict]: assert url_to_origin(url) == self.origin + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) + async with self.request_lock: if self.state == ConnectionState.PENDING: if not self.socket: @@ -94,7 +97,7 @@ async def arequest( logger.trace( "connection.arequest method=%r url=%r headers=%r", method, url, headers ) - return await self.connection.arequest(method, url, headers, stream, timeout) + return await self.connection.arequest(method, url, headers, stream, ext) async def _open_socket(self, timeout: TimeoutDict = None) -> AsyncSocketStream: scheme, hostname, port = self.origin diff --git a/httpcore/_async/connection_pool.py b/httpcore/_async/connection_pool.py index c62812aa..f7bab038 100644 --- a/httpcore/_async/connection_pool.py +++ b/httpcore/_async/connection_pool.py @@ -1,6 +1,6 @@ import warnings from ssl import SSLContext -from typing import AsyncIterator, Callable, Dict, List, Optional, Set, Tuple +from typing import AsyncIterator, Callable, Dict, List, Optional, Set, Tuple, cast from .._backends.auto import AsyncLock, AsyncSemaphore from .._backends.base import lookup_async_backend @@ -153,8 +153,8 @@ async def arequest( url: URL, headers: Headers = None, stream: AsyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, Headers, AsyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, AsyncByteStream, dict]: if url[0] not in (b"http", b"https"): scheme = url[0].decode("latin-1") raise UnsupportedProtocol(f"Unsupported URL protocol {scheme!r}") @@ -162,6 +162,8 @@ async def arequest( raise LocalProtocolError("Missing hostname in URL.") origin = url_to_origin(url) + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) await self._keepalive_sweep() @@ -190,7 +192,7 @@ async def arequest( try: response = await connection.arequest( - method, url, headers=headers, stream=stream, timeout=timeout + method, url, headers=headers, stream=stream, ext=ext ) except NewConnectionRequired: connection = None @@ -199,10 +201,11 @@ async def arequest( await self._remove_from_pool(connection) raise + status_code, headers, stream, ext = response wrapped_stream = ResponseByteStream( - response[4], connection=connection, callback=self._response_closed + stream, connection=connection, callback=self._response_closed ) - return response[0], response[1], response[2], response[3], wrapped_stream + return status_code, headers, wrapped_stream, ext async def _get_connection_from_pool( self, origin: Origin @@ -305,10 +308,8 @@ async def _keepalive_sweep(self) -> None: await connection.aclose() async def _add_to_pool( - self, connection: AsyncHTTPConnection, timeout: TimeoutDict = None + self, connection: AsyncHTTPConnection, timeout: TimeoutDict ) -> None: - timeout = {} if timeout is None else timeout - logger.trace("adding connection to pool=%r", connection) await self._connection_semaphore.acquire(timeout=timeout.get("pool", None)) async with self._thread_lock: diff --git a/httpcore/_async/http11.py b/httpcore/_async/http11.py index d0c4bf54..9d55cc62 100644 --- a/httpcore/_async/http11.py +++ b/httpcore/_async/http11.py @@ -1,5 +1,5 @@ from ssl import SSLContext -from typing import AsyncIterator, List, Tuple, Union +from typing import AsyncIterator, List, Tuple, Union, cast import h11 @@ -53,11 +53,12 @@ async def arequest( url: URL, headers: Headers = None, stream: AsyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, AsyncByteStream, dict]: headers = [] if headers is None else headers stream = PlainByteStream(b"") if stream is None else stream - timeout = {} if timeout is None else timeout + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) self.state = ConnectionState.ACTIVE @@ -73,7 +74,8 @@ async def arequest( aiterator=self._receive_response_data(timeout), aclose_func=self._response_closed, ) - return (http_version, status_code, reason_phrase, headers, response_stream) + ext = {"http_version": http_version, "reason": reason_phrase} + return (status_code, headers, response_stream, ext) async def start_tls( self, hostname: bytes, timeout: TimeoutDict = None diff --git a/httpcore/_async/http2.py b/httpcore/_async/http2.py index 1346f533..3fefd8ef 100644 --- a/httpcore/_async/http2.py +++ b/httpcore/_async/http2.py @@ -1,6 +1,5 @@ -from http import HTTPStatus from ssl import SSLContext -from typing import AsyncIterator, Dict, List, Tuple +from typing import AsyncIterator, Dict, List, Tuple, cast import h2.connection import h2.events @@ -19,13 +18,6 @@ logger = get_logger(__name__) -def get_reason_phrase(status_code: int) -> bytes: - try: - return HTTPStatus(status_code).phrase.encode("ascii") - except ValueError: - return b"" - - class AsyncHTTP2Connection(AsyncBaseHTTPConnection): READ_NUM_BYTES = 64 * 1024 CONFIG = H2Configuration(validate_inbound_headers=False) @@ -99,9 +91,10 @@ async def arequest( url: URL, headers: Headers = None, stream: AsyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]: - timeout = {} if timeout is None else timeout + ext: dict = None, + ) -> Tuple[int, Headers, AsyncByteStream, dict]: + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) async with self.init_lock: if not self.sent_connection_init: @@ -123,7 +116,7 @@ async def arequest( h2_stream = AsyncHTTP2Stream(stream_id=stream_id, connection=self) self.streams[stream_id] = h2_stream self.events[stream_id] = [] - return await h2_stream.arequest(method, url, headers, stream, timeout) + return await h2_stream.arequest(method, url, headers, stream, ext) except Exception: # noqa: PIE786 self.max_streams_semaphore.release() raise @@ -283,11 +276,12 @@ async def arequest( url: URL, headers: Headers = None, stream: AsyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, AsyncByteStream, dict]: headers = [] if headers is None else [(k.lower(), v) for (k, v) in headers] stream = PlainByteStream(b"") if stream is None else stream - timeout = {} if timeout is None else timeout + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) # Send the request. seen_headers = set(key for key, value in headers) @@ -301,12 +295,14 @@ async def arequest( # Receive the response. status_code, headers = await self.receive_response(timeout) - reason_phrase = get_reason_phrase(status_code) response_stream = AsyncIteratorByteStream( aiterator=self.body_iter(timeout), aclose_func=self._response_closed ) - return (b"HTTP/2", status_code, reason_phrase, headers, response_stream) + ext = { + "http_version": b"HTTP/2", + } + return (status_code, headers, response_stream, ext) async def send_headers( self, diff --git a/httpcore/_async/http_proxy.py b/httpcore/_async/http_proxy.py index dc3fc9d6..8a9f33c2 100644 --- a/httpcore/_async/http_proxy.py +++ b/httpcore/_async/http_proxy.py @@ -1,5 +1,6 @@ +from http import HTTPStatus from ssl import SSLContext -from typing import Tuple +from typing import Tuple, cast from .._exceptions import ProxyError from .._types import URL, Headers, TimeoutDict @@ -11,6 +12,13 @@ logger = get_logger(__name__) +def get_reason_phrase(status_code: int) -> str: + try: + return HTTPStatus(status_code).phrase + except ValueError: + return "" + + def merge_headers( default_headers: Headers = None, override_headers: Headers = None ) -> Headers: @@ -85,8 +93,8 @@ async def arequest( url: URL, headers: Headers = None, stream: AsyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, Headers, AsyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, AsyncByteStream, dict]: if self._keepalive_expiry is not None: await self._keepalive_sweep() @@ -102,7 +110,7 @@ async def arequest( url, ) return await self._forward_request( - method, url, headers=headers, stream=stream, timeout=timeout + method, url, headers=headers, stream=stream, ext=ext ) else: # By default HTTPS should be tunnelled. @@ -114,7 +122,7 @@ async def arequest( url, ) return await self._tunnel_request( - method, url, headers=headers, stream=stream, timeout=timeout + method, url, headers=headers, stream=stream, ext=ext ) async def _forward_request( @@ -123,12 +131,14 @@ async def _forward_request( url: URL, headers: Headers = None, stream: AsyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, Headers, AsyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, AsyncByteStream, dict]: """ Forwarded proxy requests include the entire URL as the HTTP target, rather than just the path. """ + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) origin = self.proxy_origin connection = await self._get_connection_from_pool(origin) @@ -136,7 +146,7 @@ async def _forward_request( connection = AsyncHTTPConnection( origin=origin, http2=self._http2, ssl_context=self._ssl_context ) - await self._add_to_pool(connection) + await self._add_to_pool(connection, timeout) # Issue a forwarded proxy request... @@ -152,21 +162,15 @@ async def _forward_request( url = self.proxy_origin + (target,) headers = merge_headers(self.proxy_headers, headers) - ( - http_version, - status_code, - reason_phrase, - headers, - stream, - ) = await connection.arequest( - method, url, headers=headers, stream=stream, timeout=timeout + (status_code, headers, stream, ext) = await connection.arequest( + method, url, headers=headers, stream=stream, ext=ext ) wrapped_stream = ResponseByteStream( stream, connection=connection, callback=self._response_closed ) - return http_version, status_code, reason_phrase, headers, wrapped_stream + return status_code, headers, wrapped_stream, ext async def _tunnel_request( self, @@ -174,12 +178,14 @@ async def _tunnel_request( url: URL, headers: Headers = None, stream: AsyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, Headers, AsyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, AsyncByteStream, dict]: """ Tunnelled proxy requests require an initial CONNECT request to establish the connection, and then send regular requests. """ + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) origin = url_to_origin(url) connection = await self._get_connection_from_pool(origin) @@ -201,19 +207,15 @@ async def _tunnel_request( connect_url = self.proxy_origin + (target,) connect_headers = [(b"Host", target), (b"Accept", b"*/*")] connect_headers = merge_headers(connect_headers, self.proxy_headers) - ( - _, - proxy_status_code, - proxy_reason_phrase, - _, - proxy_stream, - ) = await proxy_connection.arequest( - b"CONNECT", connect_url, headers=connect_headers, timeout=timeout + (proxy_status_code, _, proxy_stream, _) = await proxy_connection.arequest( + b"CONNECT", connect_url, headers=connect_headers, ext=ext ) + + proxy_reason = get_reason_phrase(proxy_status_code) logger.trace( "tunnel_response proxy_status_code=%r proxy_reason=%r ", proxy_status_code, - proxy_reason_phrase, + proxy_reason, ) # Read the response data without closing the socket async for _ in proxy_stream: @@ -221,7 +223,7 @@ async def _tunnel_request( # See if the tunnel was successfully established. if proxy_status_code < 200 or proxy_status_code > 299: - msg = "%d %s" % (proxy_status_code, proxy_reason_phrase.decode("ascii")) + msg = "%d %s" % (proxy_status_code, proxy_reason) raise ProxyError(msg) # Upgrade to TLS if required @@ -239,26 +241,20 @@ async def _tunnel_request( ssl_context=self._ssl_context, socket=proxy_connection.socket, ) - await self._add_to_pool(connection) + await self._add_to_pool(connection, timeout) # Once the connection has been established we can send requests on # it as normal. - ( - http_version, - status_code, - reason_phrase, - headers, - stream, - ) = await connection.arequest( + (status_code, headers, stream, ext) = await connection.arequest( method, url, headers=headers, stream=stream, - timeout=timeout, + ext=ext, ) wrapped_stream = ResponseByteStream( stream, connection=connection, callback=self._response_closed ) - return http_version, status_code, reason_phrase, headers, wrapped_stream + return status_code, headers, wrapped_stream, ext diff --git a/httpcore/_sync/base.py b/httpcore/_sync/base.py index 76b916e6..95a434eb 100644 --- a/httpcore/_sync/base.py +++ b/httpcore/_sync/base.py @@ -1,8 +1,8 @@ import enum from types import TracebackType -from typing import Iterator, List, Tuple, Type +from typing import Iterator, Tuple, Type -from .._types import URL, Headers, T, TimeoutDict +from .._types import URL, Headers, T class NewConnectionRequired(Exception): @@ -67,8 +67,8 @@ def request( url: URL, headers: Headers = None, stream: SyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], SyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, SyncByteStream, dict]: """ The interface for sending a single HTTP request, and returning a response. @@ -80,23 +80,17 @@ def request( * **headers** - `Optional[List[Tuple[bytes, bytes]]]` - Any HTTP headers to send with the request. * **stream** - `Optional[SyncByteStream]` - The body of the HTTP request. - * **timeout** - `Optional[Dict[str, Optional[float]]]` - A dictionary of - timeout values for I/O operations. Supported keys are "pool" for acquiring a - connection from the connection pool, "read" for reading from the connection, - "write" for writing to the connection and "connect" for opening the connection. - Values are floating point seconds. + * **ext** - `Optional[dict]` - A dictionary of optional extensions. ** Returns:** - A five-tuple of: + A four-tuple of: - * **http_version** - `bytes` - The HTTP version used by the server, - such as `b'HTTP/1.1'`. * **status_code** - `int` - The HTTP status code, such as `200`. - * **reason_phrase** - `bytes` - Any HTTP reason phrase, such as `b'OK'`. * **headers** - `List[Tuple[bytes, bytes]]` - Any HTTP headers included on the response. * **stream** - `SyncByteStream` - The body of the HTTP response. + * **ext** - `dict` - A dictionary of optional extensions. """ raise NotImplementedError() # pragma: nocover diff --git a/httpcore/_sync/connection.py b/httpcore/_sync/connection.py index 52bf5550..480acb47 100644 --- a/httpcore/_sync/connection.py +++ b/httpcore/_sync/connection.py @@ -1,5 +1,5 @@ from ssl import SSLContext -from typing import List, Optional, Tuple +from typing import Optional, Tuple, cast from .._backends.sync import SyncBackend, SyncLock, SyncSocketStream, SyncBackend from .._types import URL, Headers, Origin, TimeoutDict @@ -72,9 +72,12 @@ def request( url: URL, headers: Headers = None, stream: SyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], SyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, SyncByteStream, dict]: assert url_to_origin(url) == self.origin + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) + with self.request_lock: if self.state == ConnectionState.PENDING: if not self.socket: @@ -94,7 +97,7 @@ def request( logger.trace( "connection.request method=%r url=%r headers=%r", method, url, headers ) - return self.connection.request(method, url, headers, stream, timeout) + return self.connection.request(method, url, headers, stream, ext) def _open_socket(self, timeout: TimeoutDict = None) -> SyncSocketStream: scheme, hostname, port = self.origin diff --git a/httpcore/_sync/connection_pool.py b/httpcore/_sync/connection_pool.py index 5206c2a1..ca00099b 100644 --- a/httpcore/_sync/connection_pool.py +++ b/httpcore/_sync/connection_pool.py @@ -1,6 +1,6 @@ import warnings from ssl import SSLContext -from typing import Iterator, Callable, Dict, List, Optional, Set, Tuple +from typing import Iterator, Callable, Dict, List, Optional, Set, Tuple, cast from .._backends.sync import SyncLock, SyncSemaphore from .._backends.base import lookup_sync_backend @@ -153,8 +153,8 @@ def request( url: URL, headers: Headers = None, stream: SyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, Headers, SyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, SyncByteStream, dict]: if url[0] not in (b"http", b"https"): scheme = url[0].decode("latin-1") raise UnsupportedProtocol(f"Unsupported URL protocol {scheme!r}") @@ -162,6 +162,8 @@ def request( raise LocalProtocolError("Missing hostname in URL.") origin = url_to_origin(url) + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) self._keepalive_sweep() @@ -190,7 +192,7 @@ def request( try: response = connection.request( - method, url, headers=headers, stream=stream, timeout=timeout + method, url, headers=headers, stream=stream, ext=ext ) except NewConnectionRequired: connection = None @@ -199,10 +201,11 @@ def request( self._remove_from_pool(connection) raise + status_code, headers, stream, ext = response wrapped_stream = ResponseByteStream( - response[4], connection=connection, callback=self._response_closed + stream, connection=connection, callback=self._response_closed ) - return response[0], response[1], response[2], response[3], wrapped_stream + return status_code, headers, wrapped_stream, ext def _get_connection_from_pool( self, origin: Origin @@ -305,10 +308,8 @@ def _keepalive_sweep(self) -> None: connection.close() def _add_to_pool( - self, connection: SyncHTTPConnection, timeout: TimeoutDict = None + self, connection: SyncHTTPConnection, timeout: TimeoutDict ) -> None: - timeout = {} if timeout is None else timeout - logger.trace("adding connection to pool=%r", connection) self._connection_semaphore.acquire(timeout=timeout.get("pool", None)) with self._thread_lock: diff --git a/httpcore/_sync/http11.py b/httpcore/_sync/http11.py index 5a732519..9b227620 100644 --- a/httpcore/_sync/http11.py +++ b/httpcore/_sync/http11.py @@ -1,5 +1,5 @@ from ssl import SSLContext -from typing import Iterator, List, Tuple, Union +from typing import Iterator, List, Tuple, Union, cast import h11 @@ -53,11 +53,12 @@ def request( url: URL, headers: Headers = None, stream: SyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], SyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, SyncByteStream, dict]: headers = [] if headers is None else headers stream = PlainByteStream(b"") if stream is None else stream - timeout = {} if timeout is None else timeout + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) self.state = ConnectionState.ACTIVE @@ -73,7 +74,8 @@ def request( iterator=self._receive_response_data(timeout), close_func=self._response_closed, ) - return (http_version, status_code, reason_phrase, headers, response_stream) + ext = {"http_version": http_version, "reason": reason_phrase} + return (status_code, headers, response_stream, ext) def start_tls( self, hostname: bytes, timeout: TimeoutDict = None diff --git a/httpcore/_sync/http2.py b/httpcore/_sync/http2.py index 631fdbef..1027434b 100644 --- a/httpcore/_sync/http2.py +++ b/httpcore/_sync/http2.py @@ -1,6 +1,5 @@ -from http import HTTPStatus from ssl import SSLContext -from typing import Iterator, Dict, List, Tuple +from typing import Iterator, Dict, List, Tuple, cast import h2.connection import h2.events @@ -19,13 +18,6 @@ logger = get_logger(__name__) -def get_reason_phrase(status_code: int) -> bytes: - try: - return HTTPStatus(status_code).phrase.encode("ascii") - except ValueError: - return b"" - - class SyncHTTP2Connection(SyncBaseHTTPConnection): READ_NUM_BYTES = 64 * 1024 CONFIG = H2Configuration(validate_inbound_headers=False) @@ -99,9 +91,10 @@ def request( url: URL, headers: Headers = None, stream: SyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], SyncByteStream]: - timeout = {} if timeout is None else timeout + ext: dict = None, + ) -> Tuple[int, Headers, SyncByteStream, dict]: + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) with self.init_lock: if not self.sent_connection_init: @@ -123,7 +116,7 @@ def request( h2_stream = SyncHTTP2Stream(stream_id=stream_id, connection=self) self.streams[stream_id] = h2_stream self.events[stream_id] = [] - return h2_stream.request(method, url, headers, stream, timeout) + return h2_stream.request(method, url, headers, stream, ext) except Exception: # noqa: PIE786 self.max_streams_semaphore.release() raise @@ -283,11 +276,12 @@ def request( url: URL, headers: Headers = None, stream: SyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], SyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, SyncByteStream, dict]: headers = [] if headers is None else [(k.lower(), v) for (k, v) in headers] stream = PlainByteStream(b"") if stream is None else stream - timeout = {} if timeout is None else timeout + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) # Send the request. seen_headers = set(key for key, value in headers) @@ -301,12 +295,14 @@ def request( # Receive the response. status_code, headers = self.receive_response(timeout) - reason_phrase = get_reason_phrase(status_code) response_stream = IteratorByteStream( iterator=self.body_iter(timeout), close_func=self._response_closed ) - return (b"HTTP/2", status_code, reason_phrase, headers, response_stream) + ext = { + "http_version": b"HTTP/2", + } + return (status_code, headers, response_stream, ext) def send_headers( self, diff --git a/httpcore/_sync/http_proxy.py b/httpcore/_sync/http_proxy.py index 71805140..aa3a1ae5 100644 --- a/httpcore/_sync/http_proxy.py +++ b/httpcore/_sync/http_proxy.py @@ -1,5 +1,6 @@ +from http import HTTPStatus from ssl import SSLContext -from typing import Tuple +from typing import Tuple, cast from .._exceptions import ProxyError from .._types import URL, Headers, TimeoutDict @@ -11,6 +12,13 @@ logger = get_logger(__name__) +def get_reason_phrase(status_code: int) -> str: + try: + return HTTPStatus(status_code).phrase + except ValueError: + return "" + + def merge_headers( default_headers: Headers = None, override_headers: Headers = None ) -> Headers: @@ -85,8 +93,8 @@ def request( url: URL, headers: Headers = None, stream: SyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, Headers, SyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, SyncByteStream, dict]: if self._keepalive_expiry is not None: self._keepalive_sweep() @@ -102,7 +110,7 @@ def request( url, ) return self._forward_request( - method, url, headers=headers, stream=stream, timeout=timeout + method, url, headers=headers, stream=stream, ext=ext ) else: # By default HTTPS should be tunnelled. @@ -114,7 +122,7 @@ def request( url, ) return self._tunnel_request( - method, url, headers=headers, stream=stream, timeout=timeout + method, url, headers=headers, stream=stream, ext=ext ) def _forward_request( @@ -123,12 +131,14 @@ def _forward_request( url: URL, headers: Headers = None, stream: SyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, Headers, SyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, SyncByteStream, dict]: """ Forwarded proxy requests include the entire URL as the HTTP target, rather than just the path. """ + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) origin = self.proxy_origin connection = self._get_connection_from_pool(origin) @@ -136,7 +146,7 @@ def _forward_request( connection = SyncHTTPConnection( origin=origin, http2=self._http2, ssl_context=self._ssl_context ) - self._add_to_pool(connection) + self._add_to_pool(connection, timeout) # Issue a forwarded proxy request... @@ -152,21 +162,15 @@ def _forward_request( url = self.proxy_origin + (target,) headers = merge_headers(self.proxy_headers, headers) - ( - http_version, - status_code, - reason_phrase, - headers, - stream, - ) = connection.request( - method, url, headers=headers, stream=stream, timeout=timeout + (status_code, headers, stream, ext) = connection.request( + method, url, headers=headers, stream=stream, ext=ext ) wrapped_stream = ResponseByteStream( stream, connection=connection, callback=self._response_closed ) - return http_version, status_code, reason_phrase, headers, wrapped_stream + return status_code, headers, wrapped_stream, ext def _tunnel_request( self, @@ -174,12 +178,14 @@ def _tunnel_request( url: URL, headers: Headers = None, stream: SyncByteStream = None, - timeout: TimeoutDict = None, - ) -> Tuple[bytes, int, bytes, Headers, SyncByteStream]: + ext: dict = None, + ) -> Tuple[int, Headers, SyncByteStream, dict]: """ Tunnelled proxy requests require an initial CONNECT request to establish the connection, and then send regular requests. """ + ext = {} if ext is None else ext + timeout = cast(TimeoutDict, ext.get("timeout", {})) origin = url_to_origin(url) connection = self._get_connection_from_pool(origin) @@ -201,19 +207,15 @@ def _tunnel_request( connect_url = self.proxy_origin + (target,) connect_headers = [(b"Host", target), (b"Accept", b"*/*")] connect_headers = merge_headers(connect_headers, self.proxy_headers) - ( - _, - proxy_status_code, - proxy_reason_phrase, - _, - proxy_stream, - ) = proxy_connection.request( - b"CONNECT", connect_url, headers=connect_headers, timeout=timeout + (proxy_status_code, _, proxy_stream, _) = proxy_connection.request( + b"CONNECT", connect_url, headers=connect_headers, ext=ext ) + + proxy_reason = get_reason_phrase(proxy_status_code) logger.trace( "tunnel_response proxy_status_code=%r proxy_reason=%r ", proxy_status_code, - proxy_reason_phrase, + proxy_reason, ) # Read the response data without closing the socket for _ in proxy_stream: @@ -221,7 +223,7 @@ def _tunnel_request( # See if the tunnel was successfully established. if proxy_status_code < 200 or proxy_status_code > 299: - msg = "%d %s" % (proxy_status_code, proxy_reason_phrase.decode("ascii")) + msg = "%d %s" % (proxy_status_code, proxy_reason) raise ProxyError(msg) # Upgrade to TLS if required @@ -239,26 +241,20 @@ def _tunnel_request( ssl_context=self._ssl_context, socket=proxy_connection.socket, ) - self._add_to_pool(connection) + self._add_to_pool(connection, timeout) # Once the connection has been established we can send requests on # it as normal. - ( - http_version, - status_code, - reason_phrase, - headers, - stream, - ) = connection.request( + (status_code, headers, stream, ext) = connection.request( method, url, headers=headers, stream=stream, - timeout=timeout, + ext=ext, ) wrapped_stream = ResponseByteStream( stream, connection=connection, callback=self._response_closed ) - return http_version, status_code, reason_phrase, headers, wrapped_stream + return status_code, headers, wrapped_stream, ext diff --git a/tests/async_tests/test_interfaces.py b/tests/async_tests/test_interfaces.py index a6f0f687..e228ca4e 100644 --- a/tests/async_tests/test_interfaces.py +++ b/tests/async_tests/test_interfaces.py @@ -30,14 +30,11 @@ async def test_http_request(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -47,14 +44,11 @@ async def test_https_request(backend: str) -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -74,14 +68,11 @@ async def test_http2_request(backend: str) -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/2" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/2"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -91,14 +82,11 @@ async def test_closing_http_request(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org"), (b"connection", b"close")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert url[:3] not in http._connections # type: ignore @@ -108,27 +96,21 @@ async def test_http_request_reuse_connection(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -138,27 +120,21 @@ async def test_https_request_reuse_connection(backend: str) -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -168,14 +144,11 @@ async def test_http_request_cannot_reuse_dropped_connection(backend: str) -> Non method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore # Mock the connection as having been dropped. @@ -185,14 +158,11 @@ async def test_http_request_cannot_reuse_dropped_connection(backend: str) -> Non method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -209,14 +179,11 @@ async def test_http_proxy(proxy_server: URL, proxy_mode: str, backend: str) -> N max_connections=max_connections, backend=backend, ) as http: - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} @pytest.mark.anyio @@ -230,14 +197,11 @@ async def test_http_request_local_address(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -259,14 +223,16 @@ async def test_proxy_https_requests( max_connections=max_connections, http2=http2, ) as http: - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) _ = await read_body(stream) - assert http_version == (b"HTTP/2" if http2 else b"HTTP/1.1") + expected_ext = ( + {"http_version": b"HTTP/2"} + if http2 + else {"http_version": b"HTTP/1.1", "reason": b"OK"} + ) assert status_code == 200 - assert reason == b"OK" + assert ext == expected_ext @pytest.mark.parametrize( @@ -313,8 +279,8 @@ async def test_connection_pool_get_connection_info( url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - _, _, _, _, stream_1 = await http.arequest(method, url, headers) - _, _, _, _, stream_2 = await http.arequest(method, url, headers) + _, _, stream_1, _ = await http.arequest(method, url, headers) + _, _, stream_2, _ = await http.arequest(method, url, headers) try: stats = await http.get_connection_info() @@ -344,12 +310,9 @@ async def test_http_request_unix_domain_socket( method = b"GET" url = (b"http", b"localhost", None, b"/") headers = [(b"host", b"localhost")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) - assert http_version == b"HTTP/1.1" + status_code, headers, stream, ext = await http.arequest(method, url, headers) assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} body = await read_body(stream) assert body == b"Hello, world!" @@ -369,7 +332,7 @@ async def test_max_keepalive_connections_handled_correctly( connections_streams = [] for _ in range(connections_number): - _, _, _, _, stream = await http.arequest(method, url, headers) + _, _, stream, _ = await http.arequest(method, url, headers) connections_streams.append(stream) try: @@ -388,12 +351,9 @@ async def test_explicit_backend_name() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = await http.arequest( - method, url, headers - ) + status_code, headers, stream, ext = await http.arequest(method, url, headers) await read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore diff --git a/tests/sync_tests/test_interfaces.py b/tests/sync_tests/test_interfaces.py index 681b8742..863ca9c5 100644 --- a/tests/sync_tests/test_interfaces.py +++ b/tests/sync_tests/test_interfaces.py @@ -30,14 +30,11 @@ def test_http_request(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -47,14 +44,11 @@ def test_https_request(backend: str) -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -74,14 +68,11 @@ def test_http2_request(backend: str) -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/2" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -91,14 +82,11 @@ def test_closing_http_request(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org"), (b"connection", b"close")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert url[:3] not in http._connections # type: ignore @@ -108,27 +96,21 @@ def test_http_request_reuse_connection(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -138,27 +120,21 @@ def test_https_request_reuse_connection(backend: str) -> None: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -168,14 +144,11 @@ def test_http_request_cannot_reuse_dropped_connection(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore # Mock the connection as having been dropped. @@ -185,14 +158,11 @@ def test_http_request_cannot_reuse_dropped_connection(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -209,14 +179,11 @@ def test_http_proxy(proxy_server: URL, proxy_mode: str, backend: str) -> None: max_connections=max_connections, backend=backend, ) as http: - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} @@ -230,14 +197,11 @@ def test_http_request_local_address(backend: str) -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -259,14 +223,16 @@ def test_proxy_https_requests( max_connections=max_connections, http2=http2, ) as http: - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) _ = read_body(stream) - assert http_version == (b"HTTP/2" if http2 else b"HTTP/1.1") + expected_ext = ( + {"http_version": b"HTTP/2"} + if http2 + else {"http_version": b"HTTP/1.1", "reason": b"OK"} + ) assert status_code == 200 - assert reason == b"OK" + assert ext == expected_ext @pytest.mark.parametrize( @@ -313,8 +279,8 @@ def test_connection_pool_get_connection_info( url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] - _, _, _, _, stream_1 = http.request(method, url, headers) - _, _, _, _, stream_2 = http.request(method, url, headers) + _, _, stream_1, _ = http.request(method, url, headers) + _, _, stream_2, _ = http.request(method, url, headers) try: stats = http.get_connection_info() @@ -344,12 +310,9 @@ def test_http_request_unix_domain_socket( method = b"GET" url = (b"http", b"localhost", None, b"/") headers = [(b"host", b"localhost")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) - assert http_version == b"HTTP/1.1" + status_code, headers, stream, ext = http.request(method, url, headers) assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} body = read_body(stream) assert body == b"Hello, world!" @@ -369,7 +332,7 @@ def test_max_keepalive_connections_handled_correctly( connections_streams = [] for _ in range(connections_number): - _, _, _, _, stream = http.request(method, url, headers) + _, _, stream, _ = http.request(method, url, headers) connections_streams.append(stream) try: @@ -388,12 +351,9 @@ def test_explicit_backend_name() -> None: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] - http_version, status_code, reason, headers, stream = http.request( - method, url, headers - ) + status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) - assert http_version == b"HTTP/1.1" assert status_code == 200 - assert reason == b"OK" + assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore From 1ced330401c89a68a7ec9f2b979efef3cf7ff667 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 21 Sep 2020 17:24:56 +0100 Subject: [PATCH 06/10] Run unasync --- tests/sync_tests/test_interfaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sync_tests/test_interfaces.py b/tests/sync_tests/test_interfaces.py index 863ca9c5..17b8333a 100644 --- a/tests/sync_tests/test_interfaces.py +++ b/tests/sync_tests/test_interfaces.py @@ -72,7 +72,7 @@ def test_http2_request(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": b"HTTP/2"} assert len(http._connections[url[:3]]) == 1 # type: ignore From d2320356217e7097ca79e096548b1fb1784c5b40 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 22 Sep 2020 10:07:22 +0100 Subject: [PATCH 07/10] Use plain strings in 'ext'. Bump version to 0.11.0 --- httpcore/__init__.py | 2 +- httpcore/_async/http11.py | 5 +++- httpcore/_async/http2.py | 2 +- httpcore/_sync/http11.py | 5 +++- httpcore/_sync/http2.py | 2 +- tests/async_tests/test_interfaces.py | 36 +++++++++++++--------------- tests/sync_tests/test_interfaces.py | 36 +++++++++++++--------------- 7 files changed, 43 insertions(+), 45 deletions(-) diff --git a/httpcore/__init__.py b/httpcore/__init__.py index 0691f4ae..168023bd 100644 --- a/httpcore/__init__.py +++ b/httpcore/__init__.py @@ -51,7 +51,7 @@ "WriteError", "WriteTimeout", ] -__version__ = "0.10.2" +__version__ = "0.11.0" __locals = locals() diff --git a/httpcore/_async/http11.py b/httpcore/_async/http11.py index 9d55cc62..2e0e378d 100644 --- a/httpcore/_async/http11.py +++ b/httpcore/_async/http11.py @@ -74,7 +74,10 @@ async def arequest( aiterator=self._receive_response_data(timeout), aclose_func=self._response_closed, ) - ext = {"http_version": http_version, "reason": reason_phrase} + ext = { + "http_version": http_version.decode("ascii", errors="ignore"), + "reason": reason_phrase.decode("ascii", errors="ignore"), + } return (status_code, headers, response_stream, ext) async def start_tls( diff --git a/httpcore/_async/http2.py b/httpcore/_async/http2.py index 3fefd8ef..08ab0a2b 100644 --- a/httpcore/_async/http2.py +++ b/httpcore/_async/http2.py @@ -300,7 +300,7 @@ async def arequest( ) ext = { - "http_version": b"HTTP/2", + "http_version": "HTTP/2", } return (status_code, headers, response_stream, ext) diff --git a/httpcore/_sync/http11.py b/httpcore/_sync/http11.py index 9b227620..067d6134 100644 --- a/httpcore/_sync/http11.py +++ b/httpcore/_sync/http11.py @@ -74,7 +74,10 @@ def request( iterator=self._receive_response_data(timeout), close_func=self._response_closed, ) - ext = {"http_version": http_version, "reason": reason_phrase} + ext = { + "http_version": http_version.decode("ascii", errors="ignore"), + "reason": reason_phrase.decode("ascii", errors="ignore"), + } return (status_code, headers, response_stream, ext) def start_tls( diff --git a/httpcore/_sync/http2.py b/httpcore/_sync/http2.py index 1027434b..2d8b8d12 100644 --- a/httpcore/_sync/http2.py +++ b/httpcore/_sync/http2.py @@ -300,7 +300,7 @@ def request( ) ext = { - "http_version": b"HTTP/2", + "http_version": "HTTP/2", } return (status_code, headers, response_stream, ext) diff --git a/tests/async_tests/test_interfaces.py b/tests/async_tests/test_interfaces.py index e228ca4e..4781f9df 100644 --- a/tests/async_tests/test_interfaces.py +++ b/tests/async_tests/test_interfaces.py @@ -34,7 +34,7 @@ async def test_http_request(backend: str) -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -48,7 +48,7 @@ async def test_https_request(backend: str) -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -72,7 +72,7 @@ async def test_http2_request(backend: str) -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/2"} + assert ext == {"http_version": "HTTP/2"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -86,7 +86,7 @@ async def test_closing_http_request(backend: str) -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert url[:3] not in http._connections # type: ignore @@ -100,7 +100,7 @@ async def test_http_request_reuse_connection(backend: str) -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore method = b"GET" @@ -110,7 +110,7 @@ async def test_http_request_reuse_connection(backend: str) -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -124,7 +124,7 @@ async def test_https_request_reuse_connection(backend: str) -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore method = b"GET" @@ -134,7 +134,7 @@ async def test_https_request_reuse_connection(backend: str) -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -148,7 +148,7 @@ async def test_http_request_cannot_reuse_dropped_connection(backend: str) -> Non await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore # Mock the connection as having been dropped. @@ -162,7 +162,7 @@ async def test_http_request_cannot_reuse_dropped_connection(backend: str) -> Non await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -183,7 +183,7 @@ async def test_http_proxy(proxy_server: URL, proxy_mode: str, backend: str) -> N await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} @pytest.mark.anyio @@ -201,7 +201,7 @@ async def test_http_request_local_address(backend: str) -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -226,13 +226,9 @@ async def test_proxy_https_requests( status_code, headers, stream, ext = await http.arequest(method, url, headers) _ = await read_body(stream) - expected_ext = ( - {"http_version": b"HTTP/2"} - if http2 - else {"http_version": b"HTTP/1.1", "reason": b"OK"} - ) assert status_code == 200 - assert ext == expected_ext + assert ext["http_version"] == "HTTP/2" if http2 else "HTTP/1.1" + assert ext.get("reason", "") == "" if http2 else "OK" @pytest.mark.parametrize( @@ -312,7 +308,7 @@ async def test_http_request_unix_domain_socket( headers = [(b"host", b"localhost")] status_code, headers, stream, ext = await http.arequest(method, url, headers) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} body = await read_body(stream) assert body == b"Hello, world!" @@ -355,5 +351,5 @@ async def test_explicit_backend_name() -> None: await read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore diff --git a/tests/sync_tests/test_interfaces.py b/tests/sync_tests/test_interfaces.py index 17b8333a..718852f8 100644 --- a/tests/sync_tests/test_interfaces.py +++ b/tests/sync_tests/test_interfaces.py @@ -34,7 +34,7 @@ def test_http_request(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -48,7 +48,7 @@ def test_https_request(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -72,7 +72,7 @@ def test_http2_request(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/2"} + assert ext == {"http_version": "HTTP/2"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -86,7 +86,7 @@ def test_closing_http_request(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert url[:3] not in http._connections # type: ignore @@ -100,7 +100,7 @@ def test_http_request_reuse_connection(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore method = b"GET" @@ -110,7 +110,7 @@ def test_http_request_reuse_connection(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -124,7 +124,7 @@ def test_https_request_reuse_connection(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore method = b"GET" @@ -134,7 +134,7 @@ def test_https_request_reuse_connection(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -148,7 +148,7 @@ def test_http_request_cannot_reuse_dropped_connection(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore # Mock the connection as having been dropped. @@ -162,7 +162,7 @@ def test_http_request_cannot_reuse_dropped_connection(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -183,7 +183,7 @@ def test_http_proxy(proxy_server: URL, proxy_mode: str, backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} @@ -201,7 +201,7 @@ def test_http_request_local_address(backend: str) -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore @@ -226,13 +226,9 @@ def test_proxy_https_requests( status_code, headers, stream, ext = http.request(method, url, headers) _ = read_body(stream) - expected_ext = ( - {"http_version": b"HTTP/2"} - if http2 - else {"http_version": b"HTTP/1.1", "reason": b"OK"} - ) assert status_code == 200 - assert ext == expected_ext + assert ext["http_version"] == "HTTP/2" if http2 else "HTTP/1.1" + assert ext.get("reason", "") == "" if http2 else "OK" @pytest.mark.parametrize( @@ -312,7 +308,7 @@ def test_http_request_unix_domain_socket( headers = [(b"host", b"localhost")] status_code, headers, stream, ext = http.request(method, url, headers) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} body = read_body(stream) assert body == b"Hello, world!" @@ -355,5 +351,5 @@ def test_explicit_backend_name() -> None: read_body(stream) assert status_code == 200 - assert ext == {"http_version": b"HTTP/1.1", "reason": b"OK"} + assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore From 4c186e8cdda751bc7ed4366c8b5645b3ccf19298 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 22 Sep 2020 10:16:43 +0100 Subject: [PATCH 08/10] Version 0.11 --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba1075cc..e6cbca65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,43 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## 0.11.0 (September 22nd, 2020) + +The Transport API with 0.11.0 has a couple of significant changes. + +Firstly we've moved changed the request interface in order to +allow extensions, which will later enable us to support features +such as trailing headers, HTTP/2 server push, and CONNECT/Upgrade connections. + +The interface changes from: + +``` +request(method, url, headers, stream, timeout): + return (http_version, status_code, reason, headers, stream) +``` + +To instead including an optional dictionary of extensions on the request and response: + +``` +request(method, url, headers, stream, ext): + return (status_code, headers, stream, ext) +``` + +Secondly, the async version of `request` is now namespaced as `arequest`. + +This allows concrete transports to support both sync and async implementations +on the same class. + +### Added + +- Add curio support. (Pull #168) +- Add anyio support, with `backend="anyio"`. (Pull #169) + +### Changed + +- Update the Transport API to use 'ext' for optional extensions. (Pull ) +- Update the Transport API to use `.request` and `.arequest` so implementations can support both sync and async. (Pull #189) + ## 0.10.2 (August 20th, 2020) ### Added From da0720fd88f68690e654c53a870997e0403e5e8d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 22 Sep 2020 10:18:11 +0100 Subject: [PATCH 09/10] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6cbca65..8dead0bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,7 @@ on the same class. ### Changed -- Update the Transport API to use 'ext' for optional extensions. (Pull ) +- Update the Transport API to use 'ext' for optional extensions. (Pull #190) - Update the Transport API to use `.request` and `.arequest` so implementations can support both sync and async. (Pull #189) ## 0.10.2 (August 20th, 2020) From aac33989ff9c4535c858f5294f593e5b7b105ae3 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 22 Sep 2020 11:00:53 +0100 Subject: [PATCH 10/10] Update CHANGELOG --- CHANGELOG.md | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dead0bf..0435836e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,28 +8,44 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). The Transport API with 0.11.0 has a couple of significant changes. -Firstly we've moved changed the request interface in order to -allow extensions, which will later enable us to support features +Firstly we've moved changed the request interface in order to allow extensions, which will later enable us to support features such as trailing headers, HTTP/2 server push, and CONNECT/Upgrade connections. The interface changes from: -``` -request(method, url, headers, stream, timeout): +```python +def request(method, url, headers, stream, timeout): return (http_version, status_code, reason, headers, stream) ``` To instead including an optional dictionary of extensions on the request and response: -``` -request(method, url, headers, stream, ext): +```python +def request(method, url, headers, stream, ext): return (status_code, headers, stream, ext) ``` +Having an open-ended extensions point will allow us to add later support for various optional features, that wouldn't otherwise be supported without these API changes. + +In particular: + +* Trailing headers support. +* HTTP/2 Server Push +* sendfile. +* Exposing raw connection on CONNECT, Upgrade, HTTP/2 bi-di streaming. +* Exposing debug information out of the API, including template name, template context. + +Currently extensions are limited to: + +* request: `timeout` - Optional. Timeout dictionary. +* response: `http_version` - Optional. Include the HTTP version used on the response. +* response: `reason` - Optional. Include the reason phrase used on the response. Only valid with HTTP/1.*. + +See https://github.com/encode/httpx/issues/1274#issuecomment-694884553 for the history behind this. + Secondly, the async version of `request` is now namespaced as `arequest`. -This allows concrete transports to support both sync and async implementations -on the same class. +This allows concrete transports to support both sync and async implementations on the same class. ### Added