From 1bb05ac68abe21b4b69587aaece34c6198bc3b05 Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 20 Sep 2024 17:21:23 +0330 Subject: [PATCH 01/16] add proxy and proxy_auth varriables to ClientSession.__init__ --- aiohttp/client.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/aiohttp/client.py b/aiohttp/client.py index 9c2fd8073a3..52820bdfa8c 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -252,6 +252,8 @@ class ClientSession: "_max_line_size", "_max_field_size", "_resolve_charset", + "_default_proxy", + "_default_proxy_auth" ) def __init__( @@ -261,6 +263,8 @@ def __init__( connector: Optional[BaseConnector] = None, cookies: Optional[LooseCookies] = None, headers: Optional[LooseHeaders] = None, + proxy: Optional[StrOrURL] = None, + proxy_auth: Optional[BasicAuth] = None, skip_auto_headers: Optional[Iterable[str]] = None, auth: Optional[BasicAuth] = None, json_serialize: JSONEncoder = json.dumps, @@ -361,6 +365,15 @@ def __init__( self._resolve_charset = fallback_charset_resolver + if proxy is not None: + try: + proxy = URL(proxy) + except ValueError as e: + raise InvalidURL(proxy) from e + self._default_proxy = proxy + self._default_proxy_auth = proxy_auth + + def __init_subclass__(cls: Type["ClientSession"]) -> None: raise TypeError( "Inheritance class {} from ClientSession " @@ -480,6 +493,12 @@ async def _request( for i in skip_auto_headers: skip_headers.add(istr(i)) + if proxy is None and self._default_proxy is not None: + proxy = self._default_proxy + + if proxy_auth is None and self._default_proxy_auth is not None: + proxy_auth = self._default_proxy_auth + if proxy is not None: try: proxy = URL(proxy) From 2229a5fe478214aec771d3693744c2c12bc69013 Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 20 Sep 2024 17:35:52 +0330 Subject: [PATCH 02/16] add meshya to contributors --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 9ab093ae2a7..5ef10dd4993 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -243,6 +243,7 @@ Matthieu Hauglustaine Matthieu Rigal Matvey Tingaev Meet Mangukiya +Meshya Michael Ihnatenko Michał Górny Mikhail Burshteyn From 129e66d5e11065ac9a109038398540df7c4b6999 Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 20 Sep 2024 17:46:14 +0330 Subject: [PATCH 03/16] set auth_proxy from default if and only if proxy variable not been set in _request function --- aiohttp/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 52820bdfa8c..9c7bb4f9efb 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -495,8 +495,6 @@ async def _request( if proxy is None and self._default_proxy is not None: proxy = self._default_proxy - - if proxy_auth is None and self._default_proxy_auth is not None: proxy_auth = self._default_proxy_auth if proxy is not None: From a27b44998ed49163926203c5c03d63022a5dd804 Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 20 Sep 2024 18:14:50 +0330 Subject: [PATCH 04/16] add test unit for default proxy variables --- tests/test_client_session.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index d68d800c0e4..ad667bb7611 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -703,6 +703,17 @@ async def test_proxy_str(session: ClientSession, params: _Params) -> None: dict(allow_redirects=True, proxy="http://proxy.com", **params), ] +async def test_default_proxy(params: _Params) -> None: + session = ClientSession(proxy="http://proxy.com") + with mock.patch( + "aiohttp.client.ClientSession._request", autospec=True, spec_set=True + ) as patched: + await session.get("http://test.example.com", **params) + assert patched.called, "`ClientSession._request` not called" + assert list(patched.call_args) == [ + (session, "GET", "http://test.example.com"), + dict(allow_redirects=True, **params), + ] async def test_request_tracing( loop: asyncio.AbstractEventLoop, aiohttp_client: AiohttpClient From de664af0284a25037b56dee35baabaada4751c81 Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 20 Sep 2024 18:15:54 +0330 Subject: [PATCH 05/16] write about default variables at docs --- docs/client_advanced.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/client_advanced.rst b/docs/client_advanced.rst index 0a7093661ff..19a4c196c33 100644 --- a/docs/client_advanced.rst +++ b/docs/client_advanced.rst @@ -605,6 +605,13 @@ Authentication credentials can be passed in proxy URL:: session.get("http://python.org", proxy="http://user:pass@some.proxy.com") +And you may set default proxy:: + + proxy_auth = aiohttp.BasicAuth('user', 'pass') + async with aiohttp.ClientSession(proxy="http://proxy.com", proxy_auth=proxy_auth) as session: + async with session.get("http://python.org") as resp: + print(resp.status) + Contrary to the ``requests`` library, it won't read environment variables by default. But you can do so by passing ``trust_env=True`` into :class:`aiohttp.ClientSession` From 4a0a3265e9e0025faa262c481e7d99ee604f393f Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 20 Sep 2024 18:16:30 +0330 Subject: [PATCH 06/16] write chages to change log --- CHANGES/9175.misc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 CHANGES/9175.misc.rst diff --git a/CHANGES/9175.misc.rst b/CHANGES/9175.misc.rst new file mode 100644 index 00000000000..01cfc23c84c --- /dev/null +++ b/CHANGES/9175.misc.rst @@ -0,0 +1 @@ +Add default proxy variables to ClientSession -- by :user:`meshya`. \ No newline at end of file From 2816e39f24b0c65ae79418fb0f56808479a43396 Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 20 Sep 2024 18:19:29 +0330 Subject: [PATCH 07/16] edit changes log file name --- CHANGES/{9175.misc.rst => 9207.misc.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CHANGES/{9175.misc.rst => 9207.misc.rst} (100%) diff --git a/CHANGES/9175.misc.rst b/CHANGES/9207.misc.rst similarity index 100% rename from CHANGES/9175.misc.rst rename to CHANGES/9207.misc.rst From 25841dd70cc6534e003a6ae3e4a5715c6710c953 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 14:50:58 +0000 Subject: [PATCH 08/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- CHANGES/9207.misc.rst | 2 +- aiohttp/client.py | 3 +-- tests/test_client_session.py | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES/9207.misc.rst b/CHANGES/9207.misc.rst index 01cfc23c84c..c2cac983eb7 100644 --- a/CHANGES/9207.misc.rst +++ b/CHANGES/9207.misc.rst @@ -1 +1 @@ -Add default proxy variables to ClientSession -- by :user:`meshya`. \ No newline at end of file +Add default proxy variables to ClientSession -- by :user:`meshya`. diff --git a/aiohttp/client.py b/aiohttp/client.py index 9c7bb4f9efb..18644efeb09 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -253,7 +253,7 @@ class ClientSession: "_max_field_size", "_resolve_charset", "_default_proxy", - "_default_proxy_auth" + "_default_proxy_auth", ) def __init__( @@ -373,7 +373,6 @@ def __init__( self._default_proxy = proxy self._default_proxy_auth = proxy_auth - def __init_subclass__(cls: Type["ClientSession"]) -> None: raise TypeError( "Inheritance class {} from ClientSession " diff --git a/tests/test_client_session.py b/tests/test_client_session.py index ad667bb7611..aee8ed4994a 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -703,6 +703,7 @@ async def test_proxy_str(session: ClientSession, params: _Params) -> None: dict(allow_redirects=True, proxy="http://proxy.com", **params), ] + async def test_default_proxy(params: _Params) -> None: session = ClientSession(proxy="http://proxy.com") with mock.patch( @@ -715,6 +716,7 @@ async def test_default_proxy(params: _Params) -> None: dict(allow_redirects=True, **params), ] + async def test_request_tracing( loop: asyncio.AbstractEventLoop, aiohttp_client: AiohttpClient ) -> None: From 48928530fb12de83682b869b606835a2f513b499 Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 20 Sep 2024 18:28:02 +0330 Subject: [PATCH 09/16] edit changes log file name --- CHANGES/{9207.misc.rst => 9207.feature.rst} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename CHANGES/{9207.misc.rst => 9207.feature.rst} (95%) diff --git a/CHANGES/9207.misc.rst b/CHANGES/9207.feature.rst similarity index 95% rename from CHANGES/9207.misc.rst rename to CHANGES/9207.feature.rst index 01cfc23c84c..f6469530d96 100644 --- a/CHANGES/9207.misc.rst +++ b/CHANGES/9207.feature.rst @@ -1 +1 @@ -Add default proxy variables to ClientSession -- by :user:`meshya`. \ No newline at end of file +Add default proxy variables to ClientSession -- by :user:`meshya`. \ No newline at end of file From 53d1e39c55989763d4a69f45cbc9bfbd12c8eb34 Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 27 Sep 2024 00:21:25 +0330 Subject: [PATCH 10/16] fix test for default proxy --- aiohttp/client.py | 3 +- tests/test_client_session.py | 60 +++++++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 805e40d4884..7ebe2e321cd 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -492,8 +492,9 @@ async def _request( for i in skip_auto_headers: skip_headers.add(istr(i)) - if proxy is None and self._default_proxy is not None: + if proxy is None: proxy = self._default_proxy + if proxy_auth is None: proxy_auth = self._default_proxy_auth if proxy is not None: diff --git a/tests/test_client_session.py b/tests/test_client_session.py index aee8ed4994a..4b175e3225a 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -703,18 +703,56 @@ async def test_proxy_str(session: ClientSession, params: _Params) -> None: dict(allow_redirects=True, proxy="http://proxy.com", **params), ] +async def test_default_proxy(loop: asyncio.AbstractEventLoop) -> None: + proxy_url = URL('http://proxy.example.com') + proxy_auth = mock.Mock() + proxy_url2 = URL('http://proxy.example2.com') + proxy_auth2 = mock.Mock() + class OnCall(Exception): ... + request_class_mock = mock.Mock(side_effect=OnCall()) + session = ClientSession( + proxy=proxy_url, + proxy_auth=proxy_auth, + request_class=request_class_mock) + + assert session._default_proxy == proxy_url ,'`ClientSession._default_proxy` not set' + assert session._default_proxy_auth == proxy_auth, '`ClientSession._default_proxy_auth` not set' + + try: + await session.get( + 'http://example.com', + ) + except OnCall: + pass + + assert request_class_mock.called, "request class not called" + assert request_class_mock.call_args[1].get( + 'proxy' + )==proxy_url , '`ClientSession._request` uses default proxy not one used in ClientSession.get' + assert request_class_mock.call_args[1].get( + 'proxy_auth' + )==proxy_auth , '`ClientSession._request` uses default proxy_auth not one used in ClientSession.get' + + request_class_mock.reset_mock() + try: + await session.get( + 'http://example.com', + proxy=proxy_url2, + proxy_auth=proxy_auth2 + ) + except OnCall: + pass + + assert request_class_mock.called, "request class not called" + assert request_class_mock.call_args[1].get( + 'proxy' + )==proxy_url2 , '`ClientSession._request` uses default proxy not one used in ClientSession.get' + assert request_class_mock.call_args[1].get( + 'proxy_auth' + )==proxy_auth2 , '`ClientSession._request` uses default proxy_auth not one used in ClientSession.get' + + await session.close() -async def test_default_proxy(params: _Params) -> None: - session = ClientSession(proxy="http://proxy.com") - with mock.patch( - "aiohttp.client.ClientSession._request", autospec=True, spec_set=True - ) as patched: - await session.get("http://test.example.com", **params) - assert patched.called, "`ClientSession._request` not called" - assert list(patched.call_args) == [ - (session, "GET", "http://test.example.com"), - dict(allow_redirects=True, **params), - ] async def test_request_tracing( From 23c656a0a093280e8a1dd8d2fa3b6a046a7b1fb1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 20:53:22 +0000 Subject: [PATCH 11/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_client_session.py | 53 ++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 53068970838..6d995cab5f2 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -709,58 +709,59 @@ async def test_proxy_str(session: ClientSession, params: _Params) -> None: dict(allow_redirects=True, proxy="http://proxy.com", **params), ] + async def test_default_proxy(loop: asyncio.AbstractEventLoop) -> None: - proxy_url = URL('http://proxy.example.com') + proxy_url = URL("http://proxy.example.com") proxy_auth = mock.Mock() - proxy_url2 = URL('http://proxy.example2.com') + proxy_url2 = URL("http://proxy.example2.com") proxy_auth2 = mock.Mock() + class OnCall(Exception): ... + request_class_mock = mock.Mock(side_effect=OnCall()) session = ClientSession( - proxy=proxy_url, - proxy_auth=proxy_auth, - request_class=request_class_mock) + proxy=proxy_url, proxy_auth=proxy_auth, request_class=request_class_mock + ) - assert session._default_proxy == proxy_url ,'`ClientSession._default_proxy` not set' - assert session._default_proxy_auth == proxy_auth, '`ClientSession._default_proxy_auth` not set' + assert session._default_proxy == proxy_url, "`ClientSession._default_proxy` not set" + assert ( + session._default_proxy_auth == proxy_auth + ), "`ClientSession._default_proxy_auth` not set" - try: + try: await session.get( - 'http://example.com', + "http://example.com", ) except OnCall: pass assert request_class_mock.called, "request class not called" - assert request_class_mock.call_args[1].get( - 'proxy' - )==proxy_url , '`ClientSession._request` uses default proxy not one used in ClientSession.get' - assert request_class_mock.call_args[1].get( - 'proxy_auth' - )==proxy_auth , '`ClientSession._request` uses default proxy_auth not one used in ClientSession.get' + assert ( + request_class_mock.call_args[1].get("proxy") == proxy_url + ), "`ClientSession._request` uses default proxy not one used in ClientSession.get" + assert ( + request_class_mock.call_args[1].get("proxy_auth") == proxy_auth + ), "`ClientSession._request` uses default proxy_auth not one used in ClientSession.get" request_class_mock.reset_mock() - try: + try: await session.get( - 'http://example.com', - proxy=proxy_url2, - proxy_auth=proxy_auth2 + "http://example.com", proxy=proxy_url2, proxy_auth=proxy_auth2 ) except OnCall: pass assert request_class_mock.called, "request class not called" - assert request_class_mock.call_args[1].get( - 'proxy' - )==proxy_url2 , '`ClientSession._request` uses default proxy not one used in ClientSession.get' - assert request_class_mock.call_args[1].get( - 'proxy_auth' - )==proxy_auth2 , '`ClientSession._request` uses default proxy_auth not one used in ClientSession.get' + assert ( + request_class_mock.call_args[1].get("proxy") == proxy_url2 + ), "`ClientSession._request` uses default proxy not one used in ClientSession.get" + assert ( + request_class_mock.call_args[1].get("proxy_auth") == proxy_auth2 + ), "`ClientSession._request` uses default proxy_auth not one used in ClientSession.get" await session.close() - async def test_request_tracing( loop: asyncio.AbstractEventLoop, aiohttp_client: AiohttpClient ) -> None: From 4c567773599f51b785592ece8c1f6aeaab6962b4 Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 27 Sep 2024 00:37:45 +0330 Subject: [PATCH 12/16] remove dublicated code --- aiohttp/client.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/aiohttp/client.py b/aiohttp/client.py index 7ebe2e321cd..44366eed92c 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -365,11 +365,6 @@ def __init__( self._resolve_charset = fallback_charset_resolver - if proxy is not None: - try: - proxy = URL(proxy) - except ValueError as e: - raise InvalidURL(proxy) from e self._default_proxy = proxy self._default_proxy_auth = proxy_auth From a1ac53608cce79fcbc2c80df7f8c920ccac467ed Mon Sep 17 00:00:00 2001 From: meshya Date: Fri, 27 Sep 2024 00:42:16 +0330 Subject: [PATCH 13/16] try edit linter error --- tests/test_client_session.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 6d995cab5f2..5b1b527f7ec 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -716,7 +716,8 @@ async def test_default_proxy(loop: asyncio.AbstractEventLoop) -> None: proxy_url2 = URL("http://proxy.example2.com") proxy_auth2 = mock.Mock() - class OnCall(Exception): ... + class OnCall(Exception): + pass request_class_mock = mock.Mock(side_effect=OnCall()) session = ClientSession( From bab1e7f0a725c2aff38c78d6d2c34fbf09929832 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 27 Sep 2024 16:28:02 +0100 Subject: [PATCH 14/16] Update codeql.yml --- .github/codeql.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/codeql.yml b/.github/codeql.yml index dc4bcc4a4dc..90355ad66e0 100644 --- a/.github/codeql.yml +++ b/.github/codeql.yml @@ -1,3 +1,4 @@ query-filters: - exclude: + id: py/ineffectual-statement id: py/unsafe-cyclic-import From 7e945ff83d68a653842a93feace6a354487cf1ca Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 27 Sep 2024 17:15:47 +0100 Subject: [PATCH 15/16] Update CHANGES/9207.feature.rst --- CHANGES/9207.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES/9207.feature.rst b/CHANGES/9207.feature.rst index c2cac983eb7..d9ac55c8520 100644 --- a/CHANGES/9207.feature.rst +++ b/CHANGES/9207.feature.rst @@ -1 +1 @@ -Add default proxy variables to ClientSession -- by :user:`meshya`. +Added ``proxy`` and ``proxy_auth`` parameters to ``ClientSession`` -- by :user:`meshya`. From 1057e95932b10e738179dd333fe7953ea4fd31f5 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Fri, 27 Sep 2024 17:19:51 +0100 Subject: [PATCH 16/16] pytest.raises --- tests/test_client_session.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 5b1b527f7ec..f8efeac077d 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -729,12 +729,10 @@ class OnCall(Exception): session._default_proxy_auth == proxy_auth ), "`ClientSession._default_proxy_auth` not set" - try: + with pytest.raises(OnCall): await session.get( "http://example.com", ) - except OnCall: - pass assert request_class_mock.called, "request class not called" assert ( @@ -745,12 +743,10 @@ class OnCall(Exception): ), "`ClientSession._request` uses default proxy_auth not one used in ClientSession.get" request_class_mock.reset_mock() - try: + with pytest.raises(OnCall): await session.get( "http://example.com", proxy=proxy_url2, proxy_auth=proxy_auth2 ) - except OnCall: - pass assert request_class_mock.called, "request class not called" assert (