Merge pull request #1825 from orderbird/proxy_from_env

Proxy from environment variables

CHANGES.rst

@@ -23,6 +23,8 @@ Changes
 
 - Avoid creating TimerContext when there is no timeout to allow compatibility with Tornado. #1817 #1180
 
+- Add `proxy_from_env` to `ClientRequest` to read from environment variables. #1791
+
 
 2.0.7 (2017-04-12)
 ------------------

CONTRIBUTORS.txt

@@ -109,6 +109,7 @@ Martin Melka
 Martin Richard
 Mathias Fröjdman
 Matthieu Hauglustaine
+Matthieu Rigal
 Michael Ihnatenko
 Mikhail Kashkin
 Mikhail Lukyanchenko

aiohttp/client_reqrep.py

@@ -6,6 +6,7 @@
 import traceback
 import warnings
 from http.cookies import CookieError, Morsel
+from urllib.request import getproxies
 
 from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy
 from yarl import URL
@@ -63,7 +64,8 @@ def __init__(self, method, url, *,
                  auth=None, version=http.HttpVersion11, compress=None,
                  chunked=None, expect100=False,
                  loop=None, response_class=None,
-                 proxy=None, proxy_auth=None, timer=None):
+                 proxy=None, proxy_auth=None, proxy_from_env=False,
+                 timer=None):
 
         if loop is None:
             loop = asyncio.get_event_loop()
@@ -96,7 +98,7 @@ def __init__(self, method, url, *,
         self.update_cookies(cookies)
         self.update_content_encoding(data)
         self.update_auth(auth)
-        self.update_proxy(proxy, proxy_auth)
+        self.update_proxy(proxy, proxy_auth, proxy_from_env)
 
         self.update_body_from_data(data, skip_auto_headers)
         self.update_transfer_encoding()
@@ -290,7 +292,10 @@ def update_expect_continue(self, expect=False):
         if expect:
             self._continue = helpers.create_future(self.loop)
 
-    def update_proxy(self, proxy, proxy_auth):
+    def update_proxy(self, proxy, proxy_auth, proxy_from_env):
+        if proxy_from_env and not proxy:
+            proxy_url = getproxies().get(self.original_url.scheme)
+            proxy = URL(proxy_url) if proxy_url else None
         if proxy and not proxy.scheme == 'http':
             raise ValueError("Only http proxies are supported")
         if proxy_auth and not isinstance(proxy_auth, helpers.BasicAuth):

docs/client.rst

@@ -555,7 +555,17 @@ aiohttp supports proxy. You have to use
                               proxy="http://some.proxy.com") as resp:
            print(resp.status)
 
-it also supports proxy authorization::
+Contrary to the ``requests`` library, it won't read environment variables by
+default. But you can do so by setting :attr:`proxy_from_env` to True.
+It will use the ``getproxies()`` method from ``urllib`` and thus read the
+value of the ``$url-scheme_proxy`` variable::
+
+   async with aiohttp.ClientSession() as session:
+       async with session.get("http://python.org",
+                              proxy_from_env=True) as resp:
+           print(resp.status)
+
+It also supports proxy authorization::
 
    async with aiohttp.ClientSession() as session:
        proxy_auth = aiohttp.BasicAuth('user', 'pass')

tests/test_proxy.py

@@ -1,8 +1,10 @@
 import asyncio
 import gc
+import os
 import socket
 import unittest
 from unittest import mock
+from urllib.request import getproxies
 
 from yarl import URL
 
@@ -283,7 +285,6 @@ def test_https_connect_http_proxy_error(self, ClientRequestMock):
         proxy_resp.start = make_mocked_coro(
             mock.Mock(status=400, reason='bad request'))
 
-        connector = aiohttp.TCPConnector(loop=self.loop)
         connector = aiohttp.TCPConnector(loop=self.loop)
         connector._resolve_host = make_mocked_coro(
             [{'hostname': 'hostname', 'host': '127.0.0.1', 'port': 80,
@@ -462,3 +463,152 @@ def test_https_auth(self, ClientRequestMock):
         self.loop.run_until_complete(proxy_req.close())
         proxy_resp.close()
         self.loop.run_until_complete(req.close())
+
+    @mock.patch.dict(os.environ, {'http_proxy': 'http://proxy.example.com'})
+    @mock.patch('aiohttp.connector.ClientRequest')
+    def test_connect_env_var_ignored(self, ClientRequestMock):
+        req = ClientRequest(
+            'GET', URL('http://www.python.org'),
+            loop=self.loop)
+        self.assertIsNone(req.proxy)
+
+        # mock all the things!
+        connector = aiohttp.TCPConnector(loop=self.loop)
+        connector._resolve_host = make_mocked_coro([mock.MagicMock()])
+
+        proto = mock.Mock()
+        self.loop.create_connection = make_mocked_coro(
+            (proto.transport, proto))
+        conn = self.loop.run_until_complete(connector.connect(req))
+        self.assertEqual(req.url, URL('http://www.python.org'))
+        self.assertIs(conn._protocol, proto)
+        self.assertIs(conn.transport, proto.transport)
+
+        # we patch only the connector, it should not be called
+        ClientRequestMock.assert_not_called()
+        self.assertIn('http', getproxies())
+
+    @mock.patch.dict(os.environ, {'http_proxy': 'http://proxy.example.com'})
+    @mock.patch('aiohttp.connector.ClientRequest')
+    def test_connect_env_var_used(self, ClientRequestMock):
+        req = ClientRequest(
+            'GET', URL('http://www.python.org'),
+            proxy_from_env=True,
+            loop=self.loop
+        )
+        self.assertEqual(str(req.proxy), 'http://proxy.example.com')
+
+        # mock all the things!
+        connector = aiohttp.TCPConnector(loop=self.loop)
+        connector._resolve_host = make_mocked_coro([mock.MagicMock()])
+
+        proto = mock.Mock()
+        self.loop.create_connection = make_mocked_coro(
+            (proto.transport, proto))
+        conn = self.loop.run_until_complete(connector.connect(req))
+        self.assertEqual(req.url, URL('http://www.python.org'))
+        self.assertIs(conn._protocol, proto)
+        self.assertIs(conn.transport, proto.transport)
+
+        ClientRequestMock.assert_called_with(
+            'GET', URL('http://proxy.example.com'),
+            auth=None,
+            headers={'Host': 'www.python.org'},
+            loop=self.loop)
+        self.assertIn('http', getproxies())
+
+    @mock.patch.dict(os.environ, {'https_proxy': 'http://proxy.example.com'})
+    @mock.patch('aiohttp.connector.ClientRequest')
+    def test_connect_env_var_https_ignored(self, ClientRequestMock):
+        req = ClientRequest(
+            'GET', URL('http://www.python.org'),
+            proxy_from_env=True,
+            loop=self.loop
+        )
+        self.assertIsNone(req.proxy)
+
+        # mock all the things!
+        connector = aiohttp.TCPConnector(loop=self.loop)
+        connector._resolve_host = make_mocked_coro([mock.MagicMock()])
+
+        proto = mock.Mock()
+        self.loop.create_connection = make_mocked_coro(
+            (proto.transport, proto))
+        conn = self.loop.run_until_complete(connector.connect(req))
+        self.assertEqual(req.url, URL('http://www.python.org'))
+        self.assertIs(conn._protocol, proto)
+        self.assertIs(conn.transport, proto.transport)
+
+        # we patch only the connector, it should not be called
+        ClientRequestMock.assert_not_called()
+        self.assertIn('https', getproxies())
+        self.assertNotIn('http', getproxies())
+
+    @mock.patch.dict(os.environ, {'https_proxy': 'http://proxy.example.com'})
+    @mock.patch('aiohttp.connector.ClientRequest')
+    def test_connect_env_var_https_used(self, ClientRequestMock):
+        proxy_req = ClientRequest('GET', URL('http://proxy.example.com'),
+                                  loop=self.loop)
+        ClientRequestMock.return_value = proxy_req
+
+        proxy_resp = ClientResponse('get', URL('http://proxy.example.com'))
+        proxy_resp._loop = self.loop
+        proxy_req.send = send_mock = mock.Mock()
+        send_mock.return_value = proxy_resp
+        proxy_resp.start = make_mocked_coro(mock.Mock(status=200))
+
+        connector = aiohttp.TCPConnector(loop=self.loop)
+        connector._resolve_host = make_mocked_coro(
+            [{'hostname': 'hostname', 'host': '127.0.0.1', 'port': 80,
+              'family': socket.AF_INET, 'proto': 0, 'flags': 0}])
+
+        tr, proto = mock.Mock(), mock.Mock()
+        self.loop.create_connection = make_mocked_coro((tr, proto))
+
+        req = ClientRequest(
+            'GET', URL('https://www.python.org'),
+            proxy_from_env=True,
+            loop=self.loop,
+        )
+        self.loop.run_until_complete(connector._create_connection(req))
+
+        self.assertEqual(req.url.path, '/')
+        self.assertEqual(proxy_req.method, 'CONNECT')
+        self.assertEqual(proxy_req.url, URL('https://www.python.org'))
+        tr.close.assert_called_once_with()
+        tr.get_extra_info.assert_called_with('socket', default=None)
+
+        self.loop.run_until_complete(proxy_req.close())
+        proxy_resp.close()
+        self.loop.run_until_complete(req.close())
+        self.assertIn('https', getproxies())
+
+    @mock.patch.dict(os.environ, {'http_proxy': 'http://proxy23.example.com'})
+    @mock.patch('aiohttp.connector.ClientRequest')
+    def test_connect_env_var_not_overwriting(self, ClientRequestMock):
+        req = ClientRequest(
+            'GET', URL('http://www.python.org'),
+            proxy_from_env=True,
+            proxy=URL('http://proxy.example.com'),
+            loop=self.loop
+        )
+        self.assertEqual(str(req.proxy), 'http://proxy.example.com')
+
+        # mock all the things!
+        connector = aiohttp.TCPConnector(loop=self.loop)
+        connector._resolve_host = make_mocked_coro([mock.MagicMock()])
+
+        proto = mock.Mock()
+        self.loop.create_connection = make_mocked_coro(
+            (proto.transport, proto))
+        conn = self.loop.run_until_complete(connector.connect(req))
+        self.assertEqual(req.url, URL('http://www.python.org'))
+        self.assertIs(conn._protocol, proto)
+        self.assertIs(conn.transport, proto.transport)
+
+        ClientRequestMock.assert_called_with(
+            'GET', URL('http://proxy.example.com'),
+            auth=None,
+            headers={'Host': 'www.python.org'},
+            loop=self.loop)
+        self.assertIn('http', getproxies())