diff --git a/requirements.txt b/requirements.txt index 430120a89b6d..df02fef47b3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,10 +7,10 @@ lxml==4.6.3 pygments==2.9.0 python-dateutil==2.8.2 pyyaml==5.4.1 -httpx[http2]==0.17.1 +httpx[http2]==0.18.2 Brotli==1.0.9 uvloop==0.16.0; python_version >= '3.7' uvloop==0.14.0; python_version < '3.7' -httpx-socks[asyncio]==0.3.1 +httpx-socks[asyncio]==0.4.1 langdetect==1.0.9 setproctitle==1.2.2 diff --git a/searx/network/client.py b/searx/network/client.py index 60171e6c3d3d..1e423b9bfa6d 100644 --- a/searx/network/client.py +++ b/searx/network/client.py @@ -30,15 +30,18 @@ LOOP = None SSLCONTEXTS = {} TRANSPORT_KWARGS = { - 'backend': 'asyncio', + # use anyio : + # * https://github.com/encode/httpcore/issues/344 + # * https://github.com/encode/httpx/discussions/1511 + 'backend': 'anyio', 'trust_env': False, } # pylint: disable=protected-access async def close_connections_for_url( - connection_pool: httpcore.AsyncConnectionPool, - url: httpcore._utils.URL ): + connection_pool: httpcore.AsyncConnectionPool, url: httpcore._utils.URL +): origin = httpcore._utils.url_to_origin(url) logger.debug('Drop connections for %r', origin) @@ -63,8 +66,10 @@ def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http class AsyncHTTPTransportNoHttp(httpcore.AsyncHTTPTransport): """Block HTTP request""" - async def arequest(self, method, url, headers=None, stream=None, ext=None): - raise httpcore.UnsupportedProtocol("HTTP protocol is disabled") + async def handle_async_request( + self, method, url, headers=None, stream=None, extensions=None + ): + raise httpcore.UnsupportedProtocol('HTTP protocol is disabled') class AsyncProxyTransportFixed(AsyncProxyTransport): @@ -79,30 +84,33 @@ class AsyncProxyTransportFixed(AsyncProxyTransport): * self._response_closed(self, connection) Note: AsyncProxyTransport inherit from AsyncConnectionPool - - Note: the API is going to change on httpx 0.18.0 - see https://github.com/encode/httpx/pull/1522 """ - async def arequest(self, method, url, headers=None, stream=None, ext=None): + async def handle_async_request( + self, method, url, headers=None, stream=None, extensions=None + ): retry = 2 while retry > 0: retry -= 1 try: - return await super().arequest(method, url, headers, stream, ext) + return await super().handle_async_request( + method, url, headers=headers, stream=stream, extensions=extensions + ) except (ProxyConnectionError, ProxyTimeoutError, ProxyError) as e: - raise httpcore.ProxyError(e) + raise httpx.ProxyError(e) except OSError as e: # socket.gaierror when DNS resolution fails - raise httpcore.NetworkError(e) + raise httpx.NetworkError(e) except httpcore.RemoteProtocolError as e: # in case of httpcore.RemoteProtocolError: Server disconnected await close_connections_for_url(self, url) logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e) # retry except (httpcore.NetworkError, httpcore.ProtocolError) as e: + # workaround before anyio was the default backend: # httpcore.WriteError on HTTP/2 connection leaves a new opened stream # then each new request creates a new stream and raise the same WriteError + # most probably not necessary: https://github.com/encode/httpcore/issues/357 await close_connections_for_url(self, url) raise e @@ -110,28 +118,29 @@ async def arequest(self, method, url, headers=None, stream=None, ext=None): class AsyncHTTPTransportFixed(httpx.AsyncHTTPTransport): """Fix httpx.AsyncHTTPTransport""" - async def arequest(self, method, url, headers=None, stream=None, ext=None): + async def handle_async_request( + self, method, url, headers=None, stream=None, extensions=None + ): retry = 2 while retry > 0: retry -= 1 try: - return await super().arequest(method, url, headers, stream, ext) + return await super().handle_async_request( + method, url, headers=headers, stream=stream, extensions=extensions + ) except OSError as e: # socket.gaierror when DNS resolution fails raise httpcore.ConnectError(e) - except httpcore.CloseError as e: - # httpcore.CloseError: [Errno 104] Connection reset by peer - # raised by _keepalive_sweep() - # from https://github.com/encode/httpcore/blob/4b662b5c42378a61e54d673b4c949420102379f5/httpcore/_backends/asyncio.py#L198 # pylint: disable=line-too-long - await close_connections_for_url(self._pool, url) - logger.warning('httpcore.CloseError: retry', exc_info=e) - # retry except httpcore.RemoteProtocolError as e: # in case of httpcore.RemoteProtocolError: Server disconnected await close_connections_for_url(self._pool, url) logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e) # retry except (httpcore.ProtocolError, httpcore.NetworkError) as e: + # workaround before anyio was the default backend: + # httpcore.WriteError on HTTP/2 connection leaves a new opened stream + # then each new request creates a new stream and raise the same WriteError + # most probably not necessary: https://github.com/encode/httpcore/issues/357 await close_connections_for_url(self._pool, url) raise e @@ -206,7 +215,7 @@ def new_client( if not enable_http and (pattern == 'http' or pattern.startswith('http://')): continue if (proxy_url.startswith('socks4://') - or proxy_url.startswith('socks5://') + or proxy_url.startswith('socks5://') or proxy_url.startswith('socks5h://') ): mounts[pattern] = get_transport_for_socks_proxy(