From 44c52a4114867b3a74ade2464b0b996505967bea Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Sat, 15 Aug 2020 21:50:21 +0200 Subject: [PATCH 1/5] Drop urllib3 in favor of public gist --- docs/advanced.md | 13 +-- docs/compatibility.md | 8 ++ httpx/__init__.py | 3 - httpx/_transports/urllib3.py | 149 ----------------------------------- 4 files changed, 16 insertions(+), 157 deletions(-) delete mode 100644 httpx/_transports/urllib3.py diff --git a/docs/advanced.md b/docs/advanced.md index b579447405..9b3224194b 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -809,6 +809,8 @@ HTTPX's `Client` also accepts a `transport` argument. This argument allows you to provide a custom Transport object that will be used to perform the actual sending of the requests. +### Usage + For some advanced configuration you might need to instantiate a transport class directly, and pass it to the client instance. The `httpcore` package provides a `local_address` configuration that is only available via this @@ -832,18 +834,19 @@ do not include any default values for configuring aspects such as the connection pooling details, so you'll need to provide more explicit configuration when using this API. -HTTPX also currently ships with a transport that uses the excellent -[`urllib3` library](https://urllib3.readthedocs.io/en/latest/), which can be -used with the sync `Client`... +### urllib3 transport + +This [public gist](https://gist.github.com/florimondmanca/d56764d78d748eb9f73165da388e546e) provides a transport that uses the excellent [`urllib3` library](https://urllib3.readthedocs.io/en/latest/), and can be used with the sync `Client`... ```python >>> import httpx ->>> client = httpx.Client(transport=httpx.URLLib3Transport()) +>>> from urllib3_transport import URLLib3Transport +>>> client = httpx.Client(transport=URLLib3Transport()) >>> client.get("https://example.org") ``` -Note that you'll need to install the `urllib3` package to use `URLLib3Transport`. +### Writing custom transports A transport instance must implement the Transport API defined by [`httpcore`](https://www.encode.io/httpcore/api/). You diff --git a/docs/compatibility.md b/docs/compatibility.md index 064486b934..cd1200a5e6 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -81,3 +81,11 @@ Besides, `httpx.Request()` does not support the `auth`, `timeout`, `allow_redire ## Mocking If you need to mock HTTPX the same way that test utilities like `responses` and `requests-mock` does for `requests`, see [RESPX](https://github.com/lundberg/respx). + +## Networking layer + +`requests` defers most of its HTTP networking code to the excellent [`urllib3` library](https://urllib3.readthedocs.io/en/latest/). + +On the other hand, HTTPX uses [HTTPCore](https://github.com/encode/httpcore) as its core HTTP networking layer, which is a different project than `urllib3`. + +If you're transitioning from Requests to HTTPX, you might want to consider using HTTPX with an [`urllib3` custom transport](/advanced#urllib3-transport) at first, so that `urllib3` is retained as the underlying HTTP networking implementation, allowing you to deal with any differences in behavior as a second phase. diff --git a/httpx/__init__.py b/httpx/__init__.py index 3802439466..bfd52806ef 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -38,7 +38,6 @@ from ._models import URL, Cookies, Headers, QueryParams, Request, Response from ._status_codes import StatusCode, codes from ._transports.asgi import ASGITransport -from ._transports.urllib3 import URLLib3ProxyTransport, URLLib3Transport from ._transports.wsgi import WSGITransport __all__ = [ @@ -101,8 +100,6 @@ "TransportError", "UnsupportedProtocol", "URL", - "URLLib3ProxyTransport", - "URLLib3Transport", "WriteError", "WriteTimeout", "WSGITransport", diff --git a/httpx/_transports/urllib3.py b/httpx/_transports/urllib3.py deleted file mode 100644 index c5b7af6cc2..0000000000 --- a/httpx/_transports/urllib3.py +++ /dev/null @@ -1,149 +0,0 @@ -import socket -from typing import Iterator, List, Mapping, Optional, Tuple - -import httpcore - -from .._config import create_ssl_context -from .._content_streams import ByteStream, IteratorStream -from .._exceptions import NetworkError, map_exceptions -from .._types import CertTypes, VerifyTypes - -try: - import urllib3 - from urllib3.exceptions import MaxRetryError, SSLError -except ImportError: # pragma: nocover - urllib3 = None - - -class URLLib3Transport(httpcore.SyncHTTPTransport): - def __init__( - self, - *, - verify: VerifyTypes = True, - cert: CertTypes = None, - trust_env: bool = None, - pool_connections: int = 10, - pool_maxsize: int = 10, - pool_block: bool = False, - ): - assert ( - urllib3 is not None - ), "urllib3 must be installed in order to use URLLib3Transport" - - self.pool = urllib3.PoolManager( - ssl_context=create_ssl_context( - verify=verify, cert=cert, trust_env=trust_env, http2=False - ), - num_pools=pool_connections, - maxsize=pool_maxsize, - block=pool_block, - ) - - def request( - self, - method: bytes, - url: Tuple[bytes, bytes, Optional[int], bytes], - headers: List[Tuple[bytes, bytes]] = None, - stream: httpcore.SyncByteStream = None, - timeout: Mapping[str, Optional[float]] = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], httpcore.SyncByteStream]: - headers = [] if headers is None else headers - stream = ByteStream(b"") if stream is None else stream - timeout = {} if timeout is None else timeout - - urllib3_timeout = urllib3.util.Timeout( - connect=timeout.get("connect"), read=timeout.get("read") - ) - - chunked = False - content_length = 0 - for header_key, header_value in headers: - header_key = header_key.lower() - if header_key == b"transfer-encoding": - chunked = header_value == b"chunked" - if header_key == b"content-length": - content_length = int(header_value.decode("ascii")) - body = stream if chunked or content_length else None - - scheme, host, port, path = url - default_port = {b"http": 80, "https": 443}.get(scheme) - if port is None or port == default_port: - url_str = "%s://%s%s" % ( - scheme.decode("ascii"), - host.decode("ascii"), - path.decode("ascii"), - ) - else: - url_str = "%s://%s:%d%s" % ( - scheme.decode("ascii"), - host.decode("ascii"), - port, - path.decode("ascii"), - ) - - with map_exceptions( - { - MaxRetryError: NetworkError, - SSLError: NetworkError, - socket.error: NetworkError, - } - ): - conn = self.pool.urlopen( - method=method.decode(), - url=url_str, - headers={ - key.decode("ascii"): value.decode("ascii") for key, value in headers - }, - body=body, - redirect=False, - assert_same_host=False, - retries=0, - preload_content=False, - chunked=chunked, - timeout=urllib3_timeout, - pool_timeout=timeout.get("pool"), - ) - - def response_bytes() -> Iterator[bytes]: - with map_exceptions({socket.error: NetworkError}): - for chunk in conn.stream(4096, decode_content=False): - yield chunk - - status_code = conn.status - headers = list(conn.headers.items()) - response_stream = IteratorStream( - iterator=response_bytes(), close_func=conn.release_conn - ) - return (b"HTTP/1.1", status_code, conn.reason, headers, response_stream) - - def close(self) -> None: - self.pool.clear() - - -class URLLib3ProxyTransport(URLLib3Transport): - def __init__( - self, - *, - proxy_url: str, - proxy_headers: dict = None, - verify: VerifyTypes = True, - cert: CertTypes = None, - trust_env: bool = None, - pool_connections: int = 10, - pool_maxsize: int = 10, - pool_block: bool = False, - ): - assert ( - urllib3 is not None - ), "urllib3 must be installed in order to use URLLib3ProxyTransport" - - self.pool = urllib3.ProxyManager( - proxy_url=proxy_url, - proxy_headers=proxy_headers, - ssl_context=create_ssl_context( - verify=verify, cert=cert, trust_env=trust_env, http2=False - ), - num_pools=pool_connections, - maxsize=pool_maxsize, - block=pool_block, - ) From c83842d059f86e03772cd1bfdf5d39bc46da0d7e Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Mon, 17 Aug 2020 14:50:45 +0200 Subject: [PATCH 2/5] Drop urllib3 coverage omit --- scripts/coverage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/coverage b/scripts/coverage index a476e8b278..25a2691074 100755 --- a/scripts/coverage +++ b/scripts/coverage @@ -8,4 +8,4 @@ export SOURCE_FILES="httpx tests" set -x -${PREFIX}coverage report --omit=httpx/_transports/urllib3.py --show-missing --skip-covered --fail-under=100 +${PREFIX}coverage report --show-missing --skip-covered --fail-under=100 From d42de96b7ffb888113aa8bbb8263996b30b9ef0d Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Fri, 4 Sep 2020 22:45:44 +0200 Subject: [PATCH 3/5] Drop recommendation to use urllib3 transport during Requests migration --- docs/compatibility.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index 7b1bc970d8..daef4994a7 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -89,5 +89,3 @@ If you need to mock HTTPX the same way that test utilities like `responses` and `requests` defers most of its HTTP networking code to the excellent [`urllib3` library](https://urllib3.readthedocs.io/en/latest/). On the other hand, HTTPX uses [HTTPCore](https://github.com/encode/httpcore) as its core HTTP networking layer, which is a different project than `urllib3`. - -If you're transitioning from Requests to HTTPX, you might want to consider using HTTPX with an [`urllib3` custom transport](/advanced#urllib3-transport) at first, so that `urllib3` is retained as the underlying HTTP networking implementation, allowing you to deal with any differences in behavior as a second phase. From 4f8dd341a45d0ae0f5342bc19675c4377bc43225 Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Fri, 4 Sep 2020 22:51:40 +0200 Subject: [PATCH 4/5] Add urllib3-transport to 3p pkgs --- docs/third-party-packages.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/third-party-packages.md b/docs/third-party-packages.md index cc6a79994e..61e6e860ab 100644 --- a/docs/third-party-packages.md +++ b/docs/third-party-packages.md @@ -23,3 +23,13 @@ An asynchronous GitHub API library. Includes [HTTPX support](https://gidgethub.r [GitHub](https://github.com/lundberg/respx) - [Documentation](https://lundberg.github.io/respx/) A utility for mocking out the Python HTTPX library. + +## Gists + + + +### urllib3-transport + +[GitHub](https://gist.github.com/florimondmanca/d56764d78d748eb9f73165da388e546e) + +This public gist provides an example implementation for a [custom transport](/advanced#custom-transports) implementation on top of the battle-tested [`urllib3`](https://urllib3.readthedocs.io) library. From da4a456cf1165562c4006e48a016cce07823489a Mon Sep 17 00:00:00 2001 From: florimondmanca Date: Fri, 4 Sep 2020 22:54:31 +0200 Subject: [PATCH 5/5] Drop urllib3 from dependencies list in README / docs home page --- README.md | 1 - docs/index.md | 1 - 2 files changed, 2 deletions(-) diff --git a/README.md b/README.md index 3072daf79e..f8fe0f9b60 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,6 @@ The HTTPX project relies on these excellent libraries: * `rfc3986` - URL parsing & normalization. * `idna` - Internationalized domain name support. * `sniffio` - Async library autodetection. -* `urllib3` - Support for the `httpx.URLLib3Transport` class. *(Optional)* * `brotlipy` - Decoding for "brotli" compressed responses. *(Optional)* A huge amount of credit is due to `requests` for the API layout that diff --git a/docs/index.md b/docs/index.md index 540c9bdc2d..ddea48c43f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -114,7 +114,6 @@ The HTTPX project relies on these excellent libraries: * `rfc3986` - URL parsing & normalization. * `idna` - Internationalized domain name support. * `sniffio` - Async library autodetection. -* `urllib3` - Support for the `httpx.URLLib3Transport` class. *(Optional)* * `brotlipy` - Decoding for "brotli" compressed responses. *(Optional)* A huge amount of credit is due to `requests` for the API layout that