From 2d8ec7713c7e846984c34e1468c20c14dc76db76 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 May 2020 12:13:20 +0100 Subject: [PATCH 1/7] Pass proxy_url --- httpx/_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/httpx/_client.py b/httpx/_client.py index a9fee25397..4788bf1511 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -517,7 +517,7 @@ def init_proxy_dispatch( max_connections = pool_limits.hard_limit return httpcore.SyncHTTPProxy( - proxy_origin=proxy.url.raw[:3], + proxy_url=proxy.url.raw, proxy_headers=proxy.headers.raw, proxy_mode=proxy.mode, ssl_context=ssl_context, @@ -1055,7 +1055,7 @@ def init_proxy_dispatch( max_connections = pool_limits.hard_limit return httpcore.AsyncHTTPProxy( - proxy_origin=proxy.url.raw[:3], + proxy_url=proxy.url.raw, proxy_headers=proxy.headers.raw, proxy_mode=proxy.mode, ssl_context=ssl_context, From dfcbc9ea339dedbb5beed651e1646b70f8a6dddf Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 May 2020 12:27:51 +0100 Subject: [PATCH 2/7] Rename hard_limit/soft_limit --- httpx/_client.py | 24 ++++++++---------------- httpx/_config.py | 35 +++++++++++++++++++++++++++-------- httpx/_dispatch/urllib3.py | 16 ++++++++-------- tests/test_config.py | 8 ++++---- 4 files changed, 47 insertions(+), 36 deletions(-) diff --git a/httpx/_client.py b/httpx/_client.py index a9fee25397..e46891f31f 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -493,13 +493,11 @@ def init_dispatch( ssl_context = SSLConfig( verify=verify, cert=cert, trust_env=trust_env ).ssl_context - max_keepalive = pool_limits.soft_limit - max_connections = pool_limits.hard_limit return httpcore.SyncConnectionPool( ssl_context=ssl_context, - max_keepalive=max_keepalive, - max_connections=max_connections, + max_keepalive=pool_limits.max_keepalive, + max_connections=pool_limits.max_connections, ) def init_proxy_dispatch( @@ -513,16 +511,14 @@ def init_proxy_dispatch( ssl_context = SSLConfig( verify=verify, cert=cert, trust_env=trust_env ).ssl_context - max_keepalive = pool_limits.soft_limit - max_connections = pool_limits.hard_limit return httpcore.SyncHTTPProxy( proxy_origin=proxy.url.raw[:3], proxy_headers=proxy.headers.raw, proxy_mode=proxy.mode, ssl_context=ssl_context, - max_keepalive=max_keepalive, - max_connections=max_connections, + max_keepalive=pool_limits.max_keepalive, + max_connections=pool_limits.max_connections, ) def dispatcher_for_url(self, url: URL) -> httpcore.SyncHTTPTransport: @@ -1029,13 +1025,11 @@ def init_dispatch( ssl_context = SSLConfig( verify=verify, cert=cert, trust_env=trust_env ).ssl_context - max_keepalive = pool_limits.soft_limit - max_connections = pool_limits.hard_limit return httpcore.AsyncConnectionPool( ssl_context=ssl_context, - max_keepalive=max_keepalive, - max_connections=max_connections, + max_keepalive=pool_limits.max_keepalive, + max_connections=pool_limits.max_connections, http2=http2, ) @@ -1051,16 +1045,14 @@ def init_proxy_dispatch( ssl_context = SSLConfig( verify=verify, cert=cert, trust_env=trust_env ).ssl_context - max_keepalive = pool_limits.soft_limit - max_connections = pool_limits.hard_limit return httpcore.AsyncHTTPProxy( proxy_origin=proxy.url.raw[:3], proxy_headers=proxy.headers.raw, proxy_mode=proxy.mode, ssl_context=ssl_context, - max_keepalive=max_keepalive, - max_connections=max_connections, + max_keepalive=pool_limits.max_keepalive, + max_connections=pool_limits.max_connections, http2=http2, ) diff --git a/httpx/_config.py b/httpx/_config.py index 12478821a1..54efccd4b6 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -1,6 +1,7 @@ import os import ssl import typing +import warnings from base64 import b64encode from pathlib import Path @@ -288,29 +289,47 @@ class PoolLimits: **Parameters:** - * **soft_limit** - Allow the connection pool to maintain keep-alive connections + * **max_keepalive** - Allow the connection pool to maintain keep-alive connections below this point. - * **hard_limit** - The maximum number of concurrent connections that may be + * **max_connections** - The maximum number of concurrent connections that may be established. """ def __init__( - self, *, soft_limit: int = None, hard_limit: int = None, + self, + *, + max_keepalive: int = None, + max_connections: int = None, + soft_limit: int = None, + hard_limit: int = None, ): - self.soft_limit = soft_limit - self.hard_limit = hard_limit + self.max_keepalive = max_keepalive + self.max_connections = max_connections + if soft_limit is not None: # pragma: nocover + self.max_keepalive = soft_limit + warnings.warn( + "'soft_limit' is deprecated. Use 'max_keepalive' instead.", + DeprecationWarning, + ) + if hard_limit is not None: # pragma: nocover + self.max_connections = hard_limit + warnings.warn( + "'hard_limit' is deprecated. Use 'max_connections' instead.", + DeprecationWarning, + ) def __eq__(self, other: typing.Any) -> bool: return ( isinstance(other, self.__class__) - and self.soft_limit == other.soft_limit - and self.hard_limit == other.hard_limit + and self.max_keepalive == other.max_keepalive + and self.max_connections == other.max_connections ) def __repr__(self) -> str: class_name = self.__class__.__name__ return ( - f"{class_name}(soft_limit={self.soft_limit}, hard_limit={self.hard_limit})" + f"{class_name}(max_keepalive={self.max_keepalive}, " + f"max_connections={self.max_connections})" ) diff --git a/httpx/_dispatch/urllib3.py b/httpx/_dispatch/urllib3.py index 57c257dfa6..d88618468a 100644 --- a/httpx/_dispatch/urllib3.py +++ b/httpx/_dispatch/urllib3.py @@ -26,23 +26,23 @@ def __init__( ssl_config = SSLConfig( verify=verify, cert=cert, trust_env=trust_env, http2=False ) - hard_limit = pool_limits.hard_limit - soft_limit = pool_limits.soft_limit + max_connections = pool_limits.max_connections + max_keepalive = pool_limits.max_keepalive # Our connection pool configuration doesn't quite match up with urllib3's # controls, but we can put sensible mappings in place: - if hard_limit is None: + if max_connections is None: block = False - if soft_limit is None: + if max_keepalive is None: num_pools = 1000 maxsize = 1000 else: - num_pools = int(math.sqrt(soft_limit)) - maxsize = int(math.sqrt(soft_limit)) + num_pools = int(math.sqrt(max_keepalive)) + maxsize = int(math.sqrt(max_keepalive)) else: block = True - num_pools = int(math.sqrt(hard_limit)) - maxsize = int(math.sqrt(hard_limit)) + num_pools = int(math.sqrt(max_connections)) + maxsize = int(math.sqrt(max_connections)) self.pool = self.init_pool_manager( proxy=proxy, diff --git a/tests/test_config.py b/tests/test_config.py index f0ecf317e2..41d81916ad 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -118,13 +118,13 @@ def test_ssl_eq(): def test_limits_repr(): - limits = httpx.PoolLimits(hard_limit=100) - assert repr(limits) == "PoolLimits(soft_limit=None, hard_limit=100)" + limits = httpx.PoolLimits(max_connections=100) + assert repr(limits) == "PoolLimits(max_keepalive=None, max_connections=100)" def test_limits_eq(): - limits = httpx.PoolLimits(hard_limit=100) - assert limits == httpx.PoolLimits(hard_limit=100) + limits = httpx.PoolLimits(max_connections=100) + assert limits == httpx.PoolLimits(max_connections=100) def test_timeout_eq(): From e288af606152efa4c00973c3f4dba392883e1023 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 May 2020 12:41:00 +0100 Subject: [PATCH 3/7] Use 'warn_deprecated' function --- httpx/_config.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/httpx/_config.py b/httpx/_config.py index 54efccd4b6..3a42be5081 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -1,7 +1,6 @@ import os import ssl import typing -import warnings from base64 import b64encode from pathlib import Path @@ -9,7 +8,7 @@ from ._models import URL, Headers from ._types import CertTypes, HeaderTypes, TimeoutTypes, URLTypes, VerifyTypes -from ._utils import get_ca_bundle_from_env, get_logger +from ._utils import get_ca_bundle_from_env, get_logger, warn_deprecated DEFAULT_CIPHERS = ":".join( [ @@ -307,15 +306,13 @@ def __init__( self.max_connections = max_connections if soft_limit is not None: # pragma: nocover self.max_keepalive = soft_limit - warnings.warn( + warn_deprecated( "'soft_limit' is deprecated. Use 'max_keepalive' instead.", - DeprecationWarning, ) if hard_limit is not None: # pragma: nocover self.max_connections = hard_limit - warnings.warn( + warn_deprecated( "'hard_limit' is deprecated. Use 'max_connections' instead.", - DeprecationWarning, ) def __eq__(self, other: typing.Any) -> bool: From 5fddb6ff244ddb104e0e0f007b2e9b498ef95ddb Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 May 2020 12:42:05 +0100 Subject: [PATCH 4/7] Update PoolLimits docs --- docs/advanced.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index 4d1f1d8a78..2c1f1a6042 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -404,14 +404,14 @@ response = client.get('http://example.com/') You can control the connection pool size using the `pool_limits` keyword argument on the client. It takes instances of `httpx.PoolLimits` which define: -- `soft_limit`, number of allowable keep-alive connections, or `None` to always +- `max_keepalive`, number of allowable keep-alive connections, or `None` to always allow. (Defaults 10) -- `hard_limit`, maximum number of allowable connections, or` None` for no limits. +- `max_connections`, maximum number of allowable connections, or` None` for no limits. (Default 100) ```python -limits = httpx.PoolLimits(soft_limit=5, hard_limit=10) +limits = httpx.PoolLimits(max_keepalive=5, max_connections=10) client = httpx.Client(pool_limits=limits) ``` From fcc6a49d8a9b36a2a0262d0e32f28dc6eb36cf4a Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 May 2020 12:43:07 +0100 Subject: [PATCH 5/7] Linting --- httpx/_config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/httpx/_config.py b/httpx/_config.py index 3a42be5081..0dc7a460db 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -306,9 +306,7 @@ def __init__( self.max_connections = max_connections if soft_limit is not None: # pragma: nocover self.max_keepalive = soft_limit - warn_deprecated( - "'soft_limit' is deprecated. Use 'max_keepalive' instead.", - ) + warn_deprecated("'soft_limit' is deprecated. Use 'max_keepalive' instead.",) if hard_limit is not None: # pragma: nocover self.max_connections = hard_limit warn_deprecated( From 7bf8da772e1c95d0ac9ef2111b7593f83e75ff48 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 May 2020 13:04:17 +0100 Subject: [PATCH 6/7] Update httpcore dependancy --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4fe7e1bed2..cc6216992c 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def get_packages(package): "chardet==3.*", "idna==2.*", "rfc3986>=1.3,<2", - "httpcore>=0.8.4", + "httpcore==0.9.*", ], classifiers=[ "Development Status :: 4 - Beta", From b0a1b3bcbe107e111c43693d00caeacfc22f4a8c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 May 2020 13:10:01 +0100 Subject: [PATCH 7/7] Update port in Transport API to be 'Optional[int]' --- httpx/_dispatch/asgi.py | 2 +- httpx/_dispatch/urllib3.py | 6 +++--- httpx/_dispatch/wsgi.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/httpx/_dispatch/asgi.py b/httpx/_dispatch/asgi.py index a86969bcca..03e1b840cb 100644 --- a/httpx/_dispatch/asgi.py +++ b/httpx/_dispatch/asgi.py @@ -72,7 +72,7 @@ def __init__( async def request( self, method: bytes, - url: Tuple[bytes, bytes, int, bytes], + url: Tuple[bytes, bytes, Optional[int], bytes], headers: List[Tuple[bytes, bytes]] = None, stream: httpcore.AsyncByteStream = None, timeout: Dict[str, Optional[float]] = None, diff --git a/httpx/_dispatch/urllib3.py b/httpx/_dispatch/urllib3.py index 57c257dfa6..7ab88b9c17 100644 --- a/httpx/_dispatch/urllib3.py +++ b/httpx/_dispatch/urllib3.py @@ -80,7 +80,7 @@ def init_pool_manager( def request( self, method: bytes, - url: Tuple[bytes, bytes, int, bytes], + url: Tuple[bytes, bytes, Optional[int], bytes], headers: List[Tuple[bytes, bytes]] = None, stream: httpcore.SyncByteStream = None, timeout: Dict[str, Optional[float]] = None, @@ -104,8 +104,8 @@ def request( body = stream if chunked or content_length else None scheme, host, port, path = url - default_scheme = {80: b"http", 443: "https"}.get(port) - if scheme == default_scheme: + default_port = {b"http": 80, "https": 443}.get(scheme) + if port is None or port == default_port: url_str = "%s://%s%s" % ( scheme.decode("ascii"), host.decode("ascii"), diff --git a/httpx/_dispatch/wsgi.py b/httpx/_dispatch/wsgi.py index 9d83183fce..b9af9c40ea 100644 --- a/httpx/_dispatch/wsgi.py +++ b/httpx/_dispatch/wsgi.py @@ -63,7 +63,7 @@ def __init__( def request( self, method: bytes, - url: typing.Tuple[bytes, bytes, int, bytes], + url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], headers: typing.List[typing.Tuple[bytes, bytes]] = None, stream: httpcore.SyncByteStream = None, timeout: typing.Dict[str, typing.Optional[float]] = None,