From 6dff5d53cff5552c8aa7f4deb75a8f8b225d2a1c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 12:55:33 -0400 Subject: [PATCH 01/64] tests passing locally --- sdk/core/azure-core/dev_requirements.txt | 1 + .../async_tests/test_testserver_async.py | 37 +++++++++++++++++++ sdk/core/azure-core/tests/conftest.py | 25 ++++++++++++- sdk/core/azure-core/tests/pytest.ini | 3 ++ sdk/core/azure-core/tests/test_testserver.py | 34 +++++++++++++++++ 5 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 sdk/core/azure-core/tests/async_tests/test_testserver_async.py create mode 100644 sdk/core/azure-core/tests/pytest.ini create mode 100644 sdk/core/azure-core/tests/test_testserver.py diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index f1cabe1cb127..8269e1064860 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -7,3 +7,4 @@ opencensus-ext-threading mock; python_version < '3.3' -e ../../../tools/azure-sdk-tools -e ../../../tools/azure-devtools +git+https://github.com/iscai-msft/core.testserver#subdirectory=coretestserver \ No newline at end of file diff --git a/sdk/core/azure-core/tests/async_tests/test_testserver_async.py b/sdk/core/azure-core/tests/async_tests/test_testserver_async.py new file mode 100644 index 000000000000..7cc87341678b --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/test_testserver_async.py @@ -0,0 +1,37 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import pytest +from azure.core.pipeline.transport import HttpRequest, AioHttpTransport +"""This file does a simple call to the testserver to make sure we can use the testserver""" + +@pytest.mark.asyncio +async def test_smoke(): + request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") + async with AioHttpTransport() as sender: + response = await sender.send(request) + response.raise_for_status() + await response.load_body() + assert response.text() == "Hello, world!" \ No newline at end of file diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index 0c5e14c094c7..b4f8193c3041 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -23,14 +23,37 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +import pytest +import signal +import os +import subprocess import sys +def start_testserver(): + cmd = "FLASK_APP=coretestserver flask run" + if os.name == 'nt': #On windows, subprocess creation works without being in the shell + return subprocess.Popen(cmd.format("set")) + + return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + +def terminate_testserver(process): + if os.name == 'nt': + process.kill() + else: + os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups + +@pytest.fixture(scope="session") +def testserver(): + """Start the Autorest testserver.""" + server = start_testserver() + yield + # terminate_testserver(server) + # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): collect_ignore.append("async_tests") - # If opencensus is loadable while doing these tests, register an empty tracer to avoid this: # https://github.com/census-instrumentation/opencensus-python/issues/442 try: diff --git a/sdk/core/azure-core/tests/pytest.ini b/sdk/core/azure-core/tests/pytest.ini new file mode 100644 index 000000000000..ab17f621bd0c --- /dev/null +++ b/sdk/core/azure-core/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +usefixtures=testserver +xfail_strict=true \ No newline at end of file diff --git a/sdk/core/azure-core/tests/test_testserver.py b/sdk/core/azure-core/tests/test_testserver.py new file mode 100644 index 000000000000..1a2ba43c9dae --- /dev/null +++ b/sdk/core/azure-core/tests/test_testserver.py @@ -0,0 +1,34 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from azure.core.pipeline.transport import HttpRequest, RequestsTransport +"""This file does a simple call to the testserver to make sure we can use the testserver""" + +def test_smoke(): + request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") + with RequestsTransport() as sender: + response = sender.send(request) + response.raise_for_status() + assert response.text() == "Hello, world!" From 58c184a8f79098cb828fc2c91eb2d1b14ea364d5 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 14:05:33 -0400 Subject: [PATCH 02/64] move pytests.init to root --- sdk/core/azure-core/{tests => }/pytest.ini | 0 sdk/core/azure-core/samples/conftest.py | 9 +++++++++ 2 files changed, 9 insertions(+) rename sdk/core/azure-core/{tests => }/pytest.ini (100%) diff --git a/sdk/core/azure-core/tests/pytest.ini b/sdk/core/azure-core/pytest.ini similarity index 100% rename from sdk/core/azure-core/tests/pytest.ini rename to sdk/core/azure-core/pytest.ini diff --git a/sdk/core/azure-core/samples/conftest.py b/sdk/core/azure-core/samples/conftest.py index 6d453aed7c4c..0f55930dc4d3 100644 --- a/sdk/core/azure-core/samples/conftest.py +++ b/sdk/core/azure-core/samples/conftest.py @@ -23,8 +23,17 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +import pytest import sys +@pytest.fixture(scope="session") +def testserver(): + # dummy testserver for now + # pytest.ini needs to be in root bc we run pytest on the pipelines with just "pytest" + # because of this, samples conftest needs its own def of testserver. + # plan to change the samples tests to use testserver as well, so it's not always going to be a gross dummy fixture + yield + # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): From c7663fee90eac904d1f9778cfdd3b289619cce26 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 14:58:50 -0400 Subject: [PATCH 03/64] remove set formatting from windows run --- sdk/core/azure-core/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index b4f8193c3041..08aecea27742 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -32,7 +32,7 @@ def start_testserver(): cmd = "FLASK_APP=coretestserver flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - return subprocess.Popen(cmd.format("set")) + return subprocess.Popen(cmd) return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True From bec31ed616f5e36ed34b28cdda86476fbe885cac Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 16:21:08 -0400 Subject: [PATCH 04/64] try to fix windows testserver start --- sdk/core/azure-core/tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index 08aecea27742..f4a614f38385 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -30,11 +30,11 @@ import sys def start_testserver(): - cmd = "FLASK_APP=coretestserver flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - return subprocess.Popen(cmd) + os.environ["FLASK_APP"] = "coretestserver" + return subprocess.Popen("flask run", env=dict(os.environ)) - return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + return subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True def terminate_testserver(process): if os.name == 'nt': From 405d5e83148852e3c5c2d99c2d0377d930a821c1 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 18:08:15 -0400 Subject: [PATCH 05/64] uncomment testserver termination --- sdk/core/azure-core/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index f4a614f38385..f7e7f5cad07f 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -47,7 +47,7 @@ def testserver(): """Start the Autorest testserver.""" server = start_testserver() yield - # terminate_testserver(server) + terminate_testserver(server) # Ignore collection of async tests for Python 2 collect_ignore = [] From f8263774af9ef0f20ecbc3db9d98e4288fe2fe35 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 17 Jun 2021 18:34:56 -0400 Subject: [PATCH 06/64] add devops scripts to start testserver --- scripts/devops_tasks/end_coretestserver.py | 25 ++++++++++++++++++++ scripts/devops_tasks/start_coretestserver.py | 14 +++++++++++ sdk/core/azure-core/tests/conftest.py | 9 ++++--- sdk/core/ci.yml | 12 ++++++++++ 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 scripts/devops_tasks/end_coretestserver.py create mode 100644 scripts/devops_tasks/start_coretestserver.py diff --git a/scripts/devops_tasks/end_coretestserver.py b/scripts/devops_tasks/end_coretestserver.py new file mode 100644 index 000000000000..a900ad160939 --- /dev/null +++ b/scripts/devops_tasks/end_coretestserver.py @@ -0,0 +1,25 @@ +import os +import signal +import argparse + +def end_testserver(pid): + + if os.name == 'nt': + os.kill(pid, signal.CTRL_C_EVENT) + else: + os.killpg(os.getpgid(pid), signal.SIGTERM) # Send the signal to all the process groups + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="Stop the testserver" + ) + parser.add_argument( + "-p", + "--pid", + dest="pid", + help="The pid of the subprocess the testserver is running on", + required=True, + ) + + args = parser.parse_args() + end_testserver(int(args.pid)) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py new file mode 100644 index 000000000000..7301b7c57bb2 --- /dev/null +++ b/scripts/devops_tasks/start_coretestserver.py @@ -0,0 +1,14 @@ +import os +import subprocess + +def start_testserver(): + if os.name == 'nt': #On windows, subprocess creation works without being in the shell + os.environ["FLASK_APP"] = "coretestserver" + result = subprocess.Popen("flask run", env=dict(os.environ)) + else: + result = subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + print('##vso[task.setvariable variable=FLASK_PID]{}'.format(result.pid)) + print("This is used in the pipelines to set the FLASK_PID env var. If you want to stop this testserver, kill this PID.") + +if __name__ == "__main__": + start_testserver() diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index f7e7f5cad07f..aca1030569f4 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -45,9 +45,12 @@ def terminate_testserver(process): @pytest.fixture(scope="session") def testserver(): """Start the Autorest testserver.""" - server = start_testserver() - yield - terminate_testserver(server) + if not os.environ.get("FLASK_PID"): + server = start_testserver() + yield + terminate_testserver(server) + else: + yield # Ignore collection of async tests for Python 2 collect_ignore = [] diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index c569bf00fc25..f53d9afa927e 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -44,6 +44,18 @@ extends: safeName: azurecorecoretracingtelemetry - name: azure-common safeName: azurecommon + BeforeTestSteps: + - task: PythonScript@0 + inputs: + scriptSource: 'scripts/devops_tasks/start_coretestserver.py' + displayName: "Start CoreTestServer" + AfterTestSteps: + - task: PythonScript@0 + inputs: + scriptSource: 'scripts/devops_tasks/end_coretestserver.py' + arguments: >- + -p $(FLASK_PID) + displayName: "Shut down CoreTestServer" CondaArtifacts: - name: azure-core meta_source: conda-recipe/meta.yaml From 6ceb7950aaa22a300ac250b400fc9414be8ca651 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 17 Jun 2021 19:42:30 -0400 Subject: [PATCH 07/64] scriptSource -> scriptPath --- sdk/core/ci.yml | 72 ++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index f53d9afa927e..df0f99a423ec 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -3,64 +3,64 @@ trigger: branches: include: - - master - - main - - hotfix/* - - release/* - - restapi* + - master + - main + - hotfix/* + - release/* + - restapi* paths: include: - - sdk/core/ - - eng/ - - tools/ + - sdk/core/ + - eng/ + - tools/ pr: branches: include: - - master - - main - - feature/* - - hotfix/* - - release/* - - restapi* + - master + - main + - feature/* + - hotfix/* + - release/* + - restapi* paths: include: - - sdk/core/ - - eng/ - - tools/ + - sdk/core/ + - eng/ + - tools/ extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core Artifacts: - - name: azure-core - safeName: azurecore - - name: azure-mgmt-core - safeName: azuremgmtcore - - name: azure-core-tracing-opencensus - safeName: azurecorecoretracingopencensus - - name: azure-core-tracing-opentelemetry - safeName: azurecorecoretracingtelemetry - - name: azure-common - safeName: azurecommon + - name: azure-core + safeName: azurecore + - name: azure-mgmt-core + safeName: azuremgmtcore + - name: azure-core-tracing-opencensus + safeName: azurecorecoretracingopencensus + - name: azure-core-tracing-opentelemetry + safeName: azurecorecoretracingtelemetry + - name: azure-common + safeName: azurecommon BeforeTestSteps: - task: PythonScript@0 inputs: - scriptSource: 'scripts/devops_tasks/start_coretestserver.py' + scriptPath: "scripts/devops_tasks/start_coretestserver.py" displayName: "Start CoreTestServer" AfterTestSteps: - task: PythonScript@0 inputs: - scriptSource: 'scripts/devops_tasks/end_coretestserver.py' + scriptPath: "scripts/devops_tasks/end_coretestserver.py" arguments: >- -p $(FLASK_PID) displayName: "Shut down CoreTestServer" CondaArtifacts: - - name: azure-core - meta_source: conda-recipe/meta.yaml - common_root: azure - checkout: - - package: azure-core - checkout_path: sdk/core - version: 1.12.0 \ No newline at end of file + - name: azure-core + meta_source: conda-recipe/meta.yaml + common_root: azure + checkout: + - package: azure-core + checkout_path: sdk/core + version: 1.12.0 From 0f10e5ea1ac07863b325be897d6c57dfa6d201ee Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 18 Jun 2021 15:08:25 -0400 Subject: [PATCH 08/64] set env vars --- scripts/devops_tasks/start_coretestserver.py | 7 ++++--- sdk/core/azure-core/tests/conftest.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py index 7301b7c57bb2..e9a33760fd47 100644 --- a/scripts/devops_tasks/start_coretestserver.py +++ b/scripts/devops_tasks/start_coretestserver.py @@ -2,11 +2,12 @@ import subprocess def start_testserver(): + os.environ["FLASK_APP"] = "coretestserver" + cmd = "flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - os.environ["FLASK_APP"] = "coretestserver" - result = subprocess.Popen("flask run", env=dict(os.environ)) + result = subprocess.Popen(cmd, env=dict(os.environ)) else: - result = subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + result = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True print('##vso[task.setvariable variable=FLASK_PID]{}'.format(result.pid)) print("This is used in the pipelines to set the FLASK_PID env var. If you want to stop this testserver, kill this PID.") diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index aca1030569f4..66698b53b5a1 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -30,11 +30,12 @@ import sys def start_testserver(): + os.environ["FLASK_APP"] = "coretestserver" + cmd = "flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - os.environ["FLASK_APP"] = "coretestserver" - return subprocess.Popen("flask run", env=dict(os.environ)) + return subprocess.Popen(cmd, env=dict(os.environ)) - return subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True def terminate_testserver(process): if os.name == 'nt': From c35fdc2cb1d62c1792f86bcf95d688733822aa4c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 18 Jun 2021 19:36:08 -0400 Subject: [PATCH 09/64] add pwsh testserver --- sdk/core/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index df0f99a423ec..9af9926587c3 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -45,6 +45,9 @@ extends: - name: azure-common safeName: azurecommon BeforeTestSteps: + - pwsh: | + pip install git+https://github.com/iscai-msft/core.testserver#subdirectory=coretestserver + displayName: "Pip install CoreTestServer" - task: PythonScript@0 inputs: scriptPath: "scripts/devops_tasks/start_coretestserver.py" From f8fd247f39242f0dedfc5e2be8adba8a421651ad Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 11:00:48 -0400 Subject: [PATCH 10/64] return result --- scripts/devops_tasks/start_coretestserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py index e9a33760fd47..d38b6f963d2f 100644 --- a/scripts/devops_tasks/start_coretestserver.py +++ b/scripts/devops_tasks/start_coretestserver.py @@ -10,6 +10,7 @@ def start_testserver(): result = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True print('##vso[task.setvariable variable=FLASK_PID]{}'.format(result.pid)) print("This is used in the pipelines to set the FLASK_PID env var. If you want to stop this testserver, kill this PID.") + return result if __name__ == "__main__": start_testserver() From c4fd6d2858ec9b3748e61fb27e9e470d2b013edf Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 11:38:15 -0400 Subject: [PATCH 11/64] try retuning exit code --- scripts/devops_tasks/start_coretestserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py index d38b6f963d2f..df18e40e3537 100644 --- a/scripts/devops_tasks/start_coretestserver.py +++ b/scripts/devops_tasks/start_coretestserver.py @@ -1,4 +1,5 @@ import os +import sys import subprocess def start_testserver(): @@ -13,4 +14,5 @@ def start_testserver(): return result if __name__ == "__main__": - start_testserver() + result = start_testserver() + sys.exit(0) From b322e283612f72649a045dff32fb77aaa119bc5e Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 15:28:05 -0400 Subject: [PATCH 12/64] remove ci work, make pytest fixture module level --- sdk/core/azure-core/samples/conftest.py | 9 -- sdk/core/azure-core/tests/conftest.py | 29 +------ .../tests/testserver_tests/conftest.py | 56 +++++++++++++ .../{ => tests/testserver_tests}/pytest.ini | 0 .../{ => testserver_tests}/test_testserver.py | 0 .../test_testserver_async.py | 0 sdk/core/ci.yml | 83 ++++++++----------- 7 files changed, 91 insertions(+), 86 deletions(-) create mode 100644 sdk/core/azure-core/tests/testserver_tests/conftest.py rename sdk/core/azure-core/{ => tests/testserver_tests}/pytest.ini (100%) rename sdk/core/azure-core/tests/{ => testserver_tests}/test_testserver.py (100%) rename sdk/core/azure-core/tests/{async_tests => testserver_tests}/test_testserver_async.py (100%) diff --git a/sdk/core/azure-core/samples/conftest.py b/sdk/core/azure-core/samples/conftest.py index 0f55930dc4d3..6d453aed7c4c 100644 --- a/sdk/core/azure-core/samples/conftest.py +++ b/sdk/core/azure-core/samples/conftest.py @@ -23,17 +23,8 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -import pytest import sys -@pytest.fixture(scope="session") -def testserver(): - # dummy testserver for now - # pytest.ini needs to be in root bc we run pytest on the pipelines with just "pytest" - # because of this, samples conftest needs its own def of testserver. - # plan to change the samples tests to use testserver as well, so it's not always going to be a gross dummy fixture - yield - # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index 66698b53b5a1..0c5e14c094c7 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -23,41 +23,14 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -import pytest -import signal -import os -import subprocess import sys -def start_testserver(): - os.environ["FLASK_APP"] = "coretestserver" - cmd = "flask run" - if os.name == 'nt': #On windows, subprocess creation works without being in the shell - return subprocess.Popen(cmd, env=dict(os.environ)) - - return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True - -def terminate_testserver(process): - if os.name == 'nt': - process.kill() - else: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups - -@pytest.fixture(scope="session") -def testserver(): - """Start the Autorest testserver.""" - if not os.environ.get("FLASK_PID"): - server = start_testserver() - yield - terminate_testserver(server) - else: - yield - # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): collect_ignore.append("async_tests") + # If opencensus is loadable while doing these tests, register an empty tracer to avoid this: # https://github.com/census-instrumentation/opencensus-python/issues/442 try: diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py new file mode 100644 index 000000000000..beb636073c11 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -0,0 +1,56 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import pytest +import signal +import os +import subprocess +import sys + +def start_testserver(): + os.environ["FLASK_APP"] = "coretestserver" + cmd = "flask run" + if os.name == 'nt': #On windows, subprocess creation works without being in the shell + return subprocess.Popen(cmd, env=dict(os.environ)) + + return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + +def terminate_testserver(process): + if os.name == 'nt': + process.kill() + else: + os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups + +@pytest.fixture(scope="module") +def testserver(): + """Start the Autorest testserver.""" + server = start_testserver() + yield + terminate_testserver(server) + +# Ignore collection of async tests for Python 2 +collect_ignore = [] +if sys.version_info < (3, 5): + collect_ignore.append("*_async.py") diff --git a/sdk/core/azure-core/pytest.ini b/sdk/core/azure-core/tests/testserver_tests/pytest.ini similarity index 100% rename from sdk/core/azure-core/pytest.ini rename to sdk/core/azure-core/tests/testserver_tests/pytest.ini diff --git a/sdk/core/azure-core/tests/test_testserver.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py similarity index 100% rename from sdk/core/azure-core/tests/test_testserver.py rename to sdk/core/azure-core/tests/testserver_tests/test_testserver.py diff --git a/sdk/core/azure-core/tests/async_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/test_testserver_async.py rename to sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 9af9926587c3..c569bf00fc25 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -3,67 +3,52 @@ trigger: branches: include: - - master - - main - - hotfix/* - - release/* - - restapi* + - master + - main + - hotfix/* + - release/* + - restapi* paths: include: - - sdk/core/ - - eng/ - - tools/ + - sdk/core/ + - eng/ + - tools/ pr: branches: include: - - master - - main - - feature/* - - hotfix/* - - release/* - - restapi* + - master + - main + - feature/* + - hotfix/* + - release/* + - restapi* paths: include: - - sdk/core/ - - eng/ - - tools/ + - sdk/core/ + - eng/ + - tools/ extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core Artifacts: - - name: azure-core - safeName: azurecore - - name: azure-mgmt-core - safeName: azuremgmtcore - - name: azure-core-tracing-opencensus - safeName: azurecorecoretracingopencensus - - name: azure-core-tracing-opentelemetry - safeName: azurecorecoretracingtelemetry - - name: azure-common - safeName: azurecommon - BeforeTestSteps: - - pwsh: | - pip install git+https://github.com/iscai-msft/core.testserver#subdirectory=coretestserver - displayName: "Pip install CoreTestServer" - - task: PythonScript@0 - inputs: - scriptPath: "scripts/devops_tasks/start_coretestserver.py" - displayName: "Start CoreTestServer" - AfterTestSteps: - - task: PythonScript@0 - inputs: - scriptPath: "scripts/devops_tasks/end_coretestserver.py" - arguments: >- - -p $(FLASK_PID) - displayName: "Shut down CoreTestServer" + - name: azure-core + safeName: azurecore + - name: azure-mgmt-core + safeName: azuremgmtcore + - name: azure-core-tracing-opencensus + safeName: azurecorecoretracingopencensus + - name: azure-core-tracing-opentelemetry + safeName: azurecorecoretracingtelemetry + - name: azure-common + safeName: azurecommon CondaArtifacts: - - name: azure-core - meta_source: conda-recipe/meta.yaml - common_root: azure - checkout: - - package: azure-core - checkout_path: sdk/core - version: 1.12.0 + - name: azure-core + meta_source: conda-recipe/meta.yaml + common_root: azure + checkout: + - package: azure-core + checkout_path: sdk/core + version: 1.12.0 \ No newline at end of file From 449f42a2e99d0fec277159688ac9ee890b75ea89 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 17:32:39 -0400 Subject: [PATCH 13/64] tests working without pytest.ini --- .../azure-core/tests/testserver_tests/conftest.py | 15 +++++++++++---- .../azure-core/tests/testserver_tests/pytest.ini | 3 --- .../tests/testserver_tests/test_testserver.py | 2 ++ .../testserver_tests/test_testserver_async.py | 1 + 4 files changed, 14 insertions(+), 7 deletions(-) delete mode 100644 sdk/core/azure-core/tests/testserver_tests/pytest.ini diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index beb636073c11..cdfea8da4fb5 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -23,6 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +import time import pytest import signal import os @@ -33,9 +34,13 @@ def start_testserver(): os.environ["FLASK_APP"] = "coretestserver" cmd = "flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - return subprocess.Popen(cmd, env=dict(os.environ)) - - return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + child_process = subprocess.Popen(cmd, env=dict(os.environ)) + else: + child_process = subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) + time.sleep(1) + if child_process.returncode is not None: + raise ValueError("Didn't start!") + return child_process #On linux, have to set shell=True def terminate_testserver(process): if os.name == 'nt': @@ -43,13 +48,15 @@ def terminate_testserver(process): else: os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups -@pytest.fixture(scope="module") +@pytest.fixture() def testserver(): """Start the Autorest testserver.""" server = start_testserver() yield terminate_testserver(server) + + # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): diff --git a/sdk/core/azure-core/tests/testserver_tests/pytest.ini b/sdk/core/azure-core/tests/testserver_tests/pytest.ini deleted file mode 100644 index ab17f621bd0c..000000000000 --- a/sdk/core/azure-core/tests/testserver_tests/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -usefixtures=testserver -xfail_strict=true \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py index 1a2ba43c9dae..8cbad018c60d 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py @@ -23,8 +23,10 @@ # THE SOFTWARE. # # -------------------------------------------------------------------------- +import pytest from azure.core.pipeline.transport import HttpRequest, RequestsTransport """This file does a simple call to the testserver to make sure we can use the testserver""" +pytestmark = pytest.mark.usefixtures("testserver") def test_smoke(): request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py index 7cc87341678b..ccb5d6421b4e 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py @@ -26,6 +26,7 @@ import pytest from azure.core.pipeline.transport import HttpRequest, AioHttpTransport """This file does a simple call to the testserver to make sure we can use the testserver""" +pytestmark = pytest.mark.usefixtures("testserver") @pytest.mark.asyncio async def test_smoke(): From a021de277aeb01bd572a07a43863b460ea893dc3 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 17:36:36 -0400 Subject: [PATCH 14/64] only have testserver fixture in conftest --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 3 +-- sdk/core/azure-core/tests/testserver_tests/test_testserver.py | 2 -- .../azure-core/tests/testserver_tests/test_testserver_async.py | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index cdfea8da4fb5..237ba261c166 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -48,7 +48,7 @@ def terminate_testserver(process): else: os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups -@pytest.fixture() +@pytest.fixture(autouse=True, scope="module") def testserver(): """Start the Autorest testserver.""" server = start_testserver() @@ -56,7 +56,6 @@ def testserver(): terminate_testserver(server) - # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py index 8cbad018c60d..07bbe54c911f 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py @@ -26,8 +26,6 @@ import pytest from azure.core.pipeline.transport import HttpRequest, RequestsTransport """This file does a simple call to the testserver to make sure we can use the testserver""" -pytestmark = pytest.mark.usefixtures("testserver") - def test_smoke(): request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") with RequestsTransport() as sender: diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py index ccb5d6421b4e..7cc87341678b 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py @@ -26,7 +26,6 @@ import pytest from azure.core.pipeline.transport import HttpRequest, AioHttpTransport """This file does a simple call to the testserver to make sure we can use the testserver""" -pytestmark = pytest.mark.usefixtures("testserver") @pytest.mark.asyncio async def test_smoke(): From 7a68dd3d63f8368808f14c1bf99fa84a444f1e47 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 17:45:12 -0400 Subject: [PATCH 15/64] switch to package scope --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 237ba261c166..0827d0c68d3a 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -48,7 +48,7 @@ def terminate_testserver(process): else: os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups -@pytest.fixture(autouse=True, scope="module") +@pytest.fixture(autouse=True, scope="package") def testserver(): """Start the Autorest testserver.""" server = start_testserver() From df42f157c29e746b0ade8e469c826ae1569e61d7 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 17:53:53 -0400 Subject: [PATCH 16/64] unite testserver setting --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 0827d0c68d3a..12bcda2c02e7 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -31,12 +31,11 @@ import sys def start_testserver(): - os.environ["FLASK_APP"] = "coretestserver" - cmd = "flask run" + cmd = "FLASK_APP=coretestserver flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell child_process = subprocess.Popen(cmd, env=dict(os.environ)) else: - child_process = subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) + child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) time.sleep(1) if child_process.returncode is not None: raise ValueError("Didn't start!") From da62f4b8413cb7569118ca9dc2fececdedd897e6 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 11:08:44 -0400 Subject: [PATCH 17/64] switch to environment variables --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 12bcda2c02e7..6c7258d13ba5 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -31,15 +31,17 @@ import sys def start_testserver(): - cmd = "FLASK_APP=coretestserver flask run" + os.environ["FLASK_APP"] = "coretestserver" + cmd = "flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell child_process = subprocess.Popen(cmd, env=dict(os.environ)) else: - child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) + #On linux, have to set shell=True + child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid, env=dict(os.environ)) time.sleep(1) if child_process.returncode is not None: raise ValueError("Didn't start!") - return child_process #On linux, have to set shell=True + return child_process def terminate_testserver(process): if os.name == 'nt': From e06ebb4bc553f9c251252a73dd114863e6a380c9 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 15:47:41 -0400 Subject: [PATCH 18/64] see what happens if we don't kill testserver --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 6c7258d13ba5..ecfb5f8c6c5a 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -54,7 +54,6 @@ def testserver(): """Start the Autorest testserver.""" server = start_testserver() yield - terminate_testserver(server) # Ignore collection of async tests for Python 2 From 04ddbd0830a973ea7b66a8f3f92a17092a37455e Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 17:08:36 -0400 Subject: [PATCH 19/64] cycle through ports --- .../tests/testserver_tests/conftest.py | 29 ++++++++++++++++++- .../tests/testserver_tests/test_testserver.py | 5 ++-- .../testserver_tests/test_testserver_async.py | 4 +-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index ecfb5f8c6c5a..86916f40e2da 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -23,16 +23,42 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +from re import T import time import pytest import signal import os import subprocess import sys +import random +import http.client +import urllib + +def is_port_open(port_num): + conn = http.client.HTTPSConnection("localhost:{}".format(port_num)) + try: + conn.request("GET", "/health") + return False + except ConnectionRefusedError: + return True + +def get_port(): + count = 3 + for _ in range(count): + port_num = random.randrange(3000, 5000) + if is_port_open(port_num): + return port_num + raise TypeError("Tried {} times, can't find an open port".format(count)) + +@pytest.fixture +def port(): + return os.environ["FLASK_PORT"] def start_testserver(): + port = get_port() os.environ["FLASK_APP"] = "coretestserver" - cmd = "flask run" + os.environ["FLASK_PORT"] = str(port) + cmd = "flask run -p {}".format(port) if os.name == 'nt': #On windows, subprocess creation works without being in the shell child_process = subprocess.Popen(cmd, env=dict(os.environ)) else: @@ -54,6 +80,7 @@ def testserver(): """Start the Autorest testserver.""" server = start_testserver() yield + terminate_testserver(server) # Ignore collection of async tests for Python 2 diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py index 07bbe54c911f..a31d449fbd53 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py @@ -26,8 +26,9 @@ import pytest from azure.core.pipeline.transport import HttpRequest, RequestsTransport """This file does a simple call to the testserver to make sure we can use the testserver""" -def test_smoke(): - request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") + +def test_smoke(port): + request = HttpRequest(method="GET", url="http://localhost:{}/basic/string".format(port)) with RequestsTransport() as sender: response = sender.send(request) response.raise_for_status() diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py index 7cc87341678b..623033080bd1 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py @@ -28,8 +28,8 @@ """This file does a simple call to the testserver to make sure we can use the testserver""" @pytest.mark.asyncio -async def test_smoke(): - request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") +async def test_smoke(port): + request = HttpRequest(method="GET", url="http://localhost:{}/basic/string".format(port)) async with AioHttpTransport() as sender: response = await sender.send(request) response.raise_for_status() From e860e8aa2caa72c25cd98dd201cf2811dcef2a3c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 17:12:17 -0400 Subject: [PATCH 20/64] remove scripts --- scripts/devops_tasks/end_coretestserver.py | 25 -------------------- scripts/devops_tasks/start_coretestserver.py | 18 -------------- 2 files changed, 43 deletions(-) delete mode 100644 scripts/devops_tasks/end_coretestserver.py delete mode 100644 scripts/devops_tasks/start_coretestserver.py diff --git a/scripts/devops_tasks/end_coretestserver.py b/scripts/devops_tasks/end_coretestserver.py deleted file mode 100644 index a900ad160939..000000000000 --- a/scripts/devops_tasks/end_coretestserver.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import signal -import argparse - -def end_testserver(pid): - - if os.name == 'nt': - os.kill(pid, signal.CTRL_C_EVENT) - else: - os.killpg(os.getpgid(pid), signal.SIGTERM) # Send the signal to all the process groups - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="Stop the testserver" - ) - parser.add_argument( - "-p", - "--pid", - dest="pid", - help="The pid of the subprocess the testserver is running on", - required=True, - ) - - args = parser.parse_args() - end_testserver(int(args.pid)) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py deleted file mode 100644 index df18e40e3537..000000000000 --- a/scripts/devops_tasks/start_coretestserver.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import sys -import subprocess - -def start_testserver(): - os.environ["FLASK_APP"] = "coretestserver" - cmd = "flask run" - if os.name == 'nt': #On windows, subprocess creation works without being in the shell - result = subprocess.Popen(cmd, env=dict(os.environ)) - else: - result = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True - print('##vso[task.setvariable variable=FLASK_PID]{}'.format(result.pid)) - print("This is used in the pipelines to set the FLASK_PID env var. If you want to stop this testserver, kill this PID.") - return result - -if __name__ == "__main__": - result = start_testserver() - sys.exit(0) From 2f39ad2292122f5d677a5e13e5b5dfed763e7888 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 19:17:58 -0400 Subject: [PATCH 21/64] allow 2.7 compatibility --- .../azure-core/tests/testserver_tests/conftest.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 86916f40e2da..c2dec634874f 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -31,15 +31,17 @@ import subprocess import sys import random -import http.client -import urllib +try: + import http.client as httpclient +except ImportError: + import httplib as httpclient def is_port_open(port_num): - conn = http.client.HTTPSConnection("localhost:{}".format(port_num)) + conn = httpclient.HTTPSConnection("localhost:{}".format(port_num)) try: conn.request("GET", "/health") return False - except ConnectionRefusedError: + except Exception: return True def get_port(): @@ -84,6 +86,6 @@ def testserver(): # Ignore collection of async tests for Python 2 -collect_ignore = [] +collect_ignore_glob = [] if sys.version_info < (3, 5): - collect_ignore.append("*_async.py") + collect_ignore_glob.append("*_async.py") From f503b2cdaf5698786970694941b79beee8f448ec Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 21:41:04 -0400 Subject: [PATCH 22/64] wait longer for pypy --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index c2dec634874f..1c7bfe590c54 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -66,10 +66,12 @@ def start_testserver(): else: #On linux, have to set shell=True child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid, env=dict(os.environ)) - time.sleep(1) - if child_process.returncode is not None: - raise ValueError("Didn't start!") - return child_process + count = 5 + for _ in range(count): + time.sleep(1) + if not is_port_open(port): + return child_process + raise ValueError("Didn't start!") def terminate_testserver(process): if os.name == 'nt': From 8ccce3407ce06960e6ff2e3556a6f30878c89472 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 23 Jun 2021 11:45:51 -0400 Subject: [PATCH 23/64] increase sleep to 2 for pypy --- .../azure-core/tests/testserver_tests/conftest.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 1c7bfe590c54..8ddf8a27ab03 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -23,7 +23,6 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from re import T import time import pytest import signal @@ -41,7 +40,7 @@ def is_port_open(port_num): try: conn.request("GET", "/health") return False - except Exception: + except Exception as e: return True def get_port(): @@ -66,12 +65,10 @@ def start_testserver(): else: #On linux, have to set shell=True child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid, env=dict(os.environ)) - count = 5 - for _ in range(count): - time.sleep(1) - if not is_port_open(port): - return child_process - raise ValueError("Didn't start!") + time.sleep(2) + if not child_process.returncode is None: + raise ValueError("Didn't start!") + return child_process def terminate_testserver(process): if os.name == 'nt': From 30c6e39ba424eab3fa4f637826dedc30b2b16ede Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 13:07:04 -0400 Subject: [PATCH 24/64] move core testserver into tests --- sdk/core/azure-core/dev_requirements.txt | 2 +- .../coretestserver/coretestserver/__init__.py | 33 +++++++ .../coretestserver/test_routes/__init__.py | 24 +++++ .../coretestserver/test_routes/basic.py | 66 +++++++++++++ .../coretestserver/test_routes/encoding.py | 92 +++++++++++++++++++ .../coretestserver/test_routes/errors.py | 28 ++++++ .../coretestserver/test_routes/helpers.py | 12 +++ .../coretestserver/test_routes/multipart.py | 88 ++++++++++++++++++ .../coretestserver/test_routes/streams.py | 38 ++++++++ .../coretestserver/test_routes/urlencoded.py | 26 ++++++ .../coretestserver/test_routes/xml_route.py | 46 ++++++++++ .../testserver_tests/coretestserver/setup.py | 35 +++++++ 12 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index 8269e1064860..27a3bb04ce64 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -7,4 +7,4 @@ opencensus-ext-threading mock; python_version < '3.3' -e ../../../tools/azure-sdk-tools -e ../../../tools/azure-devtools -git+https://github.com/iscai-msft/core.testserver#subdirectory=coretestserver \ No newline at end of file +-e tests/testserver_tests/coretestserver \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py new file mode 100644 index 000000000000..63560847a01f --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py @@ -0,0 +1,33 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +from flask import Flask, Response +from .test_routes import ( + basic_api, + encoding_api, + errors_api, + streams_api, + urlencoded_api, + multipart_api, + xml_api +) + +app = Flask(__name__) +app.register_blueprint(basic_api, url_prefix="/basic") +app.register_blueprint(encoding_api, url_prefix="/encoding") +app.register_blueprint(errors_api, url_prefix="/errors") +app.register_blueprint(streams_api, url_prefix="/streams") +app.register_blueprint(urlencoded_api, url_prefix="/urlencoded") +app.register_blueprint(multipart_api, url_prefix="/multipart") +app.register_blueprint(xml_api, url_prefix="/xml") + +@app.route('/health', methods=['GET']) +def latin_1_charset_utf8(): + return Response(status=200) + +if __name__ == "__main__": + app.run(debug=True) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py new file mode 100644 index 000000000000..82f4e7ac4566 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py @@ -0,0 +1,24 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +from .basic import basic_api +from .encoding import encoding_api +from .errors import errors_api +from .multipart import multipart_api +from .streams import streams_api +from .urlencoded import urlencoded_api +from .xml_route import xml_api + +__all__ = [ + "basic_api", + "encoding_api", + "errors_api", + "multipart_api", + "streams_api", + "urlencoded_api", + "xml_api", +] diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py new file mode 100644 index 000000000000..4b7d5ae92ad4 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py @@ -0,0 +1,66 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +from flask import ( + Response, + Blueprint, + request +) + +basic_api = Blueprint('basic_api', __name__) + +@basic_api.route('/string', methods=['GET']) +def string(): + return Response( + "Hello, world!", status=200, mimetype="text/plain" + ) + +@basic_api.route('/lines', methods=['GET']) +def lines(): + return Response( + "Hello,\nworld!", status=200, mimetype="text/plain" + ) + +@basic_api.route("/bytes", methods=['GET']) +def bytes(): + return Response( + "Hello, world!".encode(), status=200, mimetype="text/plain" + ) + +@basic_api.route("/html", methods=['GET']) +def html(): + return Response( + "Hello, world!", status=200, mimetype="text/html" + ) + +@basic_api.route("/json", methods=['GET']) +def json(): + return Response( + '{"greeting": "hello", "recipient": "world"}', status=200, mimetype="application/json" + ) + +@basic_api.route("/complicated-json", methods=['POST']) +def complicated_json(): + # thanks to Sean Kane for this test! + assert request.json['EmptyByte'] == '' + assert request.json['EmptyUnicode'] == '' + assert request.json['SpacesOnlyByte'] == ' ' + assert request.json['SpacesOnlyUnicode'] == ' ' + assert request.json['SpacesBeforeByte'] == ' Text' + assert request.json['SpacesBeforeUnicode'] == ' Text' + assert request.json['SpacesAfterByte'] == 'Text ' + assert request.json['SpacesAfterUnicode'] == 'Text ' + assert request.json['SpacesBeforeAndAfterByte'] == ' Text ' + assert request.json['SpacesBeforeAndAfterUnicode'] == ' Text ' + assert request.json['啊齄丂狛'] == 'ꀕ' + assert request.json['RowKey'] == 'test2' + assert request.json['啊齄丂狛狜'] == 'hello' + assert request.json["singlequote"] == "a''''b" + assert request.json["doublequote"] == 'a""""b' + assert request.json["None"] == None + + return Response(status=200) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py new file mode 100644 index 000000000000..12224e568ee5 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py @@ -0,0 +1,92 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from json import dumps +from flask import ( + Response, + Blueprint, +) + +encoding_api = Blueprint('encoding_api', __name__) + +@encoding_api.route('/latin-1', methods=['GET']) +def latin_1(): + r = Response( + "Latin 1: ÿ".encode("latin-1"), status=200 + ) + r.headers["Content-Type"] = "text/plain; charset=latin-1" + return r + +@encoding_api.route('/latin-1-with-utf-8', methods=['GET']) +def latin_1_charset_utf8(): + r = Response( + "Latin 1: ÿ".encode("latin-1"), status=200 + ) + r.headers["Content-Type"] = "text/plain; charset=utf-8" + return r + +@encoding_api.route('/no-charset', methods=['GET']) +def latin_1_no_charset(): + r = Response( + "Hello, world!", status=200 + ) + r.headers["Content-Type"] = "text/plain" + return r + +@encoding_api.route('/iso-8859-1', methods=['GET']) +def iso_8859_1(): + r = Response( + "Accented: Österreich".encode("iso-8859-1"), status=200 + ) + r.headers["Content-Type"] = "text/plain" + return r + +@encoding_api.route('/emoji', methods=['GET']) +def emoji(): + r = Response( + "👩", status=200 + ) + return r + +@encoding_api.route('/emoji-family-skin-tone-modifier', methods=['GET']) +def emoji_family_skin_tone_modifier(): + r = Response( + "👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987", status=200 + ) + return r + +@encoding_api.route('/korean', methods=['GET']) +def korean(): + r = Response( + "아가", status=200 + ) + return r + +@encoding_api.route('/json', methods=['GET']) +def json(): + data = {"greeting": "hello", "recipient": "world"} + content = dumps(data).encode("utf-16") + r = Response( + content, status=200 + ) + r.headers["Content-Type"] = "application/json; charset=utf-16" + return r + +@encoding_api.route('/invalid-codec-name', methods=['GET']) +def invalid_codec_name(): + r = Response( + "おはようございます。".encode("utf-8"), status=200 + ) + r.headers["Content-Type"] = "text/plain; charset=invalid-codec-name" + return r + +@encoding_api.route('/no-charset', methods=['GET']) +def no_charset(): + r = Response( + "Hello, world!", status=200 + ) + r.headers["Content-Type"] = "text/plain" + return r diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py new file mode 100644 index 000000000000..221f598e063a --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py @@ -0,0 +1,28 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from flask import ( + Response, + Blueprint, +) + +errors_api = Blueprint('errors_api', __name__) + +@errors_api.route('/403', methods=['GET']) +def get_403(): + return Response(status=403) + +@errors_api.route('/500', methods=['GET']) +def get_500(): + return Response(status=500) + +@errors_api.route('/stream', methods=['GET']) +def get_stream(): + class StreamingBody: + def __iter__(self): + yield b"Hello, " + yield b"world!" + return Response(StreamingBody(), status=500) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py new file mode 100644 index 000000000000..46680f65d3f9 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py @@ -0,0 +1,12 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +def assert_with_message(param_name, expected_value, actual_value): + assert expected_value == actual_value, "Expected '{}' to be '{}', got '{}'".format( + param_name, expected_value, actual_value + ) + +__all__ = ["assert_with_message"] diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py new file mode 100644 index 000000000000..236496673a2f --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py @@ -0,0 +1,88 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from copy import copy +from flask import ( + Response, + Blueprint, + request, +) +from .helpers import assert_with_message + +multipart_api = Blueprint('multipart_api', __name__) + +multipart_header_start = "multipart/form-data; boundary=" + +# NOTE: the flask behavior is different for aiohttp and requests +# in requests, we see the file content through request.form +# in aiohttp, we see the file through request.files + +@multipart_api.route('/basic', methods=['POST']) +def basic(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + if request.files: + # aiohttp + assert_with_message("content length", 258, request.content_length) + assert_with_message("num files", 1, len(request.files)) + assert_with_message("has file named fileContent", True, bool(request.files.get('fileContent'))) + file_content = request.files['fileContent'] + assert_with_message("file content type", "application/octet-stream", file_content.content_type) + assert_with_message("file content length", 14, file_content.content_length) + assert_with_message("filename", "fileContent", file_content.filename) + assert_with_message("has content disposition header", True, bool(file_content.headers.get("Content-Disposition"))) + assert_with_message( + "content disposition", + 'form-data; name="fileContent"; filename="fileContent"; filename*=utf-8\'\'fileContent', + file_content.headers["Content-Disposition"] + ) + elif request.form: + # requests + assert_with_message("content length", 184, request.content_length) + assert_with_message("fileContent", "", request.form["fileContent"]) + else: + return Response(status=400) # should be either of these + return Response(status=200) + +@multipart_api.route('/data-and-files', methods=['POST']) +def data_and_files(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + assert_with_message("message", "Hello, world!", request.form["message"]) + assert_with_message("message", "", request.form["fileContent"]) + return Response(status=200) + +@multipart_api.route('/data-and-files-tuple', methods=['POST']) +def data_and_files_tuple(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + assert_with_message("message", ["abc"], request.form["message"]) + assert_with_message("message", [""], request.form["fileContent"]) + return Response(status=200) + +@multipart_api.route('/non-seekable-filelike', methods=['POST']) +def non_seekable_filelike(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + if request.files: + # aiohttp + len_files = len(request.files) + assert_with_message("num files", 1, len_files) + # assert_with_message("content length", 258, request.content_length) + assert_with_message("num files", 1, len(request.files)) + assert_with_message("has file named file", True, bool(request.files.get('file'))) + file = request.files['file'] + assert_with_message("file content type", "application/octet-stream", file.content_type) + assert_with_message("file content length", 14, file.content_length) + assert_with_message("filename", "file", file.filename) + assert_with_message("has content disposition header", True, bool(file.headers.get("Content-Disposition"))) + assert_with_message( + "content disposition", + 'form-data; name="fileContent"; filename="fileContent"; filename*=utf-8\'\'fileContent', + file.headers["Content-Disposition"] + ) + elif request.form: + # requests + assert_with_message("num files", 1, len(request.form)) + else: + return Response(status=400) + return Response(status=200) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py new file mode 100644 index 000000000000..1aeb7c05cc21 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py @@ -0,0 +1,38 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from flask import ( + Response, + Blueprint, +) + +streams_api = Blueprint('streams_api', __name__) + +class StreamingBody: + def __iter__(self): + yield b"Hello, " + yield b"world!" + + +def streaming_body(): + yield b"Hello, " + yield b"world!" + +def stream_json_error(): + yield '{"error": {"code": "BadRequest", ' + yield' "message": "You made a bad request"}}' + +@streams_api.route('/basic', methods=['GET']) +def basic(): + return Response(streaming_body(), status=200) + +@streams_api.route('/iterable', methods=['GET']) +def iterable(): + return Response(StreamingBody(), status=200) + +@streams_api.route('/error', methods=['GET']) +def error(): + return Response(stream_json_error(), status=400) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py new file mode 100644 index 000000000000..4ea2bdd2795d --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py @@ -0,0 +1,26 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from flask import ( + Response, + Blueprint, + request, +) +from .helpers import assert_with_message + +urlencoded_api = Blueprint('urlencoded_api', __name__) + +@urlencoded_api.route('/pet/add/', methods=['POST']) +def basic(pet_id): + assert_with_message("pet_id", "1", pet_id) + assert_with_message("content type", "application/x-www-form-urlencoded", request.content_type) + assert_with_message("content length", 47, request.content_length) + assert len(request.form) == 4 + assert_with_message("pet_type", "dog", request.form["pet_type"]) + assert_with_message("pet_food", "meat", request.form["pet_food"]) + assert_with_message("name", "Fido", request.form["name"]) + assert_with_message("pet_age", "42", request.form["pet_age"]) + return Response(status=200) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py new file mode 100644 index 000000000000..c19aed97b6b5 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py @@ -0,0 +1,46 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +import xml.etree.ElementTree as ET +from flask import ( + Response, + Blueprint, + request, +) +from .helpers import assert_with_message + +xml_api = Blueprint('xml_api', __name__) + +@xml_api.route('/basic', methods=['GET', 'PUT']) +def basic(): + basic_body = """ + + + Wake up to WonderWidgets! + + + Overview + Why WonderWidgets are great + + Who buys WonderWidgets + +""" + + if request.method == 'GET': + return Response(basic_body, status=200) + elif request.method == 'PUT': + assert_with_message("content length", str(len(request.data)), request.headers["Content-Length"]) + parsed_xml = ET.fromstring(request.data.decode("utf-8")) + assert_with_message("tag", "slideshow", parsed_xml.tag) + attributes = parsed_xml.attrib + assert_with_message("title attribute", "Sample Slide Show", attributes['title']) + assert_with_message("date attribute", "Date of publication", attributes['date']) + assert_with_message("author attribute", "Yours Truly", attributes['author']) + return Response(status=200) + return Response("You have passed in method '{}' that is not 'GET' or 'PUT'".format(request.method), status=400) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py new file mode 100644 index 000000000000..a43288221498 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from setuptools import setup, find_packages + +version = "1.0.0b1" + +setup( + name="coretestserver", + version=version, + include_package_data=True, + description='Testserver for Python Core', + long_description='Testserver for Python Core', + license='MIT License', + author='Microsoft Corporation', + author_email='azpysdkhelp@microsoft.com', + url='https://github.com/iscai-msft/core.testserver', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'License :: OSI Approved :: MIT License', + ], + packages=find_packages(), + install_requires=[ + "flask" + ] +) From 20e593a8e28f53653d49a6b19510aa51c57002ad Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 13:29:05 -0400 Subject: [PATCH 25/64] switch to urllib requesting to see if port open --- .../tests/testserver_tests/conftest.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 8ddf8a27ab03..10a99fb3ce21 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -30,16 +30,12 @@ import subprocess import sys import random -try: - import http.client as httpclient -except ImportError: - import httplib as httpclient +from six.moves import urllib -def is_port_open(port_num): - conn = httpclient.HTTPSConnection("localhost:{}".format(port_num)) +def is_port_available(port_num): + req = urllib.request.Request("http://localhost:{}/health".format(port_num)) try: - conn.request("GET", "/health") - return False + return urllib.request.urlopen(req).code != 200 except Exception as e: return True @@ -47,7 +43,7 @@ def get_port(): count = 3 for _ in range(count): port_num = random.randrange(3000, 5000) - if is_port_open(port_num): + if is_port_available(port_num): return port_num raise TypeError("Tried {} times, can't find an open port".format(count)) @@ -65,10 +61,12 @@ def start_testserver(): else: #On linux, have to set shell=True child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid, env=dict(os.environ)) - time.sleep(2) - if not child_process.returncode is None: - raise ValueError("Didn't start!") - return child_process + count = 5 + for _ in range(count): + if not is_port_available(port): + return child_process + time.sleep(1) + raise ValueError("Didn't start!") def terminate_testserver(process): if os.name == 'nt': From 1483f8cd7765f713f468069e205ba2b86f7415cb Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 13:57:01 -0400 Subject: [PATCH 26/64] ignore coretestserver readme --- eng/.docsettings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml index 9b2268542ee7..940dcaddd1bc 100644 --- a/eng/.docsettings.yml +++ b/eng/.docsettings.yml @@ -93,6 +93,7 @@ known_content_issues: - ['sdk/containerregistry/azure-containerregistry/swagger/README.md', '#4554'] - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554'] - ['sdk/attestation/azure-security-attestation/swagger/README.md', '#4554'] + - ['sdk/core/azure-core/tests/testserver_tests/coretestserver/README.md', '#4554'] # common. - ['sdk/appconfiguration/azure-appconfiguration/README.md', 'common'] From aabe4097c5a20936ec5cde223b4bad4a5b057512 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 18:27:49 -0400 Subject: [PATCH 27/64] initial commit for rest layer --- sdk/core/azure-core/CHANGELOG.md | 8 +- .../azure-core/azure/core/rest/__init__.py | 56 ++ .../azure-core/azure/core/rest/_helpers.py | 283 ++++++++++ .../azure/core/rest/_helpers_py3.py | 47 ++ sdk/core/azure-core/azure/core/rest/_rest.py | 386 ++++++++++++++ .../azure-core/azure/core/rest/_rest_py3.py | 501 ++++++++++++++++++ 6 files changed, 1280 insertions(+), 1 deletion(-) create mode 100644 sdk/core/azure-core/azure/core/rest/__init__.py create mode 100644 sdk/core/azure-core/azure/core/rest/_helpers.py create mode 100644 sdk/core/azure-core/azure/core/rest/_helpers_py3.py create mode 100644 sdk/core/azure-core/azure/core/rest/_rest.py create mode 100644 sdk/core/azure-core/azure/core/rest/_rest_py3.py diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 10e242697b12..f7f28e4b55af 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -1,6 +1,12 @@ # Release History -## 1.15.1 (Unreleased) +## 1.16.0 (Unreleased) + +### New Features + +- Add new provisional module `azure.core.rest`. `azure.core.rest` is our new public simple HTTP library in `azure.core` that users will use to create requests, and consume responses. +- Add new provisional methods `send_request` onto the `azure.core.PipelineClient` and `azure.core.AsyncPipelineClient`. This method takes in +requests and sends them through our pipelines. ## 1.15.0 (2021-06-04) diff --git a/sdk/core/azure-core/azure/core/rest/__init__.py b/sdk/core/azure-core/azure/core/rest/__init__.py new file mode 100644 index 000000000000..ed79c71b858c --- /dev/null +++ b/sdk/core/azure-core/azure/core/rest/__init__.py @@ -0,0 +1,56 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +try: + from ._rest_py3 import ( + HttpRequest, + HttpResponse, + _HttpResponseBase, + ) +except (SyntaxError, ImportError): + from ._rest import ( # type: ignore + HttpRequest, + HttpResponse, + _HttpResponseBase, + ) + +__all__ = [ + "HttpRequest", + "HttpResponse", + "_HttpResponseBase", +] + +try: + from ._rest_py3 import ( # pylint: disable=unused-import + AsyncHttpResponse, + _AsyncContextManager, + ) + __all__.extend([ + "AsyncHttpResponse", + "_AsyncContextManager", + ]) + +except (SyntaxError, ImportError): + pass diff --git a/sdk/core/azure-core/azure/core/rest/_helpers.py b/sdk/core/azure-core/azure/core/rest/_helpers.py new file mode 100644 index 000000000000..61b41d14e9c0 --- /dev/null +++ b/sdk/core/azure-core/azure/core/rest/_helpers.py @@ -0,0 +1,283 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import os +import codecs +from enum import Enum +from json import dumps +import collections +from typing import ( + Optional, + Union, + Mapping, + Sequence, + List, + Tuple, + IO, + Any, + Dict, + Iterable, + Iterator, + cast, +) +import xml.etree.ElementTree as ET +import six +try: + from urlparse import urlparse # type: ignore +except ImportError: + from urllib.parse import urlparse + +################################### TYPES SECTION ######################### + +PrimitiveData = Optional[Union[str, int, float, bool]] + + +ParamsType = Dict[str, Union[PrimitiveData, Sequence[PrimitiveData]]] + +HeadersType = Dict[str, str] + +FileContent = Union[str, bytes, IO[str], IO[bytes]] +FileType = Union[ + Tuple[Optional[str], FileContent], +] + +FilesType = Union[ + Dict[str, FileType], + Sequence[Tuple[str, FileType]] +] + +ContentTypeBase = Union[str, bytes, Iterable[bytes]] + +class HttpVerbs(str, Enum): + GET = "GET" + PUT = "PUT" + POST = "POST" + HEAD = "HEAD" + PATCH = "PATCH" + DELETE = "DELETE" + MERGE = "MERGE" + +########################### ERRORS SECTION ################################# + + + +########################### HELPER SECTION ################################# + +def _verify_data_object(name, value): + if not isinstance(name, str): + raise TypeError( + "Invalid type for data name. Expected str, got {}: {}".format( + type(name), name + ) + ) + if value is not None and not isinstance(value, (str, bytes, int, float)): + raise TypeError( + "Invalid type for data value. Expected primitive type, got {}: {}".format( + type(name), name + ) + ) + +def _format_data(data): + # type: (Union[str, IO]) -> Union[Tuple[None, str], Tuple[Optional[str], IO, str]] + """Format field data according to whether it is a stream or + a string for a form-data request. + + :param data: The request field data. + :type data: str or file-like object. + """ + if hasattr(data, "read"): + data = cast(IO, data) + data_name = None + try: + if data.name[0] != "<" and data.name[-1] != ">": + data_name = os.path.basename(data.name) + except (AttributeError, TypeError): + pass + return (data_name, data, "application/octet-stream") + return (None, cast(str, data)) + +def set_urlencoded_body(data): + body = {} + for f, d in data.items(): + if not d: + continue + if isinstance(d, list): + for item in d: + _verify_data_object(f, item) + else: + _verify_data_object(f, d) + body[f] = d + return { + "Content-Type": "application/x-www-form-urlencoded" + }, body + +def set_multipart_body(files): + formatted_files = { + f: _format_data(d) for f, d in files.items() if d is not None + } + return {}, formatted_files + +def set_xml_body(content): + headers = {} + bytes_content = ET.tostring(content, encoding="utf8") + body = bytes_content.replace(b"encoding='utf8'", b"encoding='utf-8'") + if body: + headers["Content-Length"] = str(len(body)) + return headers, body + +def _shared_set_content_body(content): + # type: (Any) -> Tuple[HeadersType, Optional[ContentTypeBase]] + headers = {} # type: HeadersType + + if isinstance(content, ET.Element): + # XML body + return set_xml_body(content) + if isinstance(content, (str, bytes)): + headers = {} + body = content + if isinstance(content, six.string_types): + headers["Content-Type"] = "text/plain" + if body: + headers["Content-Length"] = str(len(body)) + return headers, body + if isinstance(content, collections.Iterable): + return {}, content + return headers, None + +def set_content_body(content): + headers, body = _shared_set_content_body(content) + if body is not None: + return headers, body + raise TypeError( + "Unexpected type for 'content': '{}'. ".format(type(content)) + + "We expect 'content' to either be str, bytes, or an Iterable" + ) + +def set_json_body(json): + # type: (Any) -> Tuple[Dict[str, str], Any] + body = dumps(json) + return { + "Content-Type": "application/json", + "Content-Length": str(len(body)) + }, body + +def format_parameters(url, params): + """Format parameters into a valid query string. + It's assumed all parameters have already been quoted as + valid URL strings. + + :param dict params: A dictionary of parameters. + """ + query = urlparse(url).query + if query: + url = url.partition("?")[0] + existing_params = { + p[0]: p[-1] for p in [p.partition("=") for p in query.split("&")] + } + params.update(existing_params) + query_params = [] + for k, v in params.items(): + if isinstance(v, list): + for w in v: + if w is None: + raise ValueError("Query parameter {} cannot be None".format(k)) + query_params.append("{}={}".format(k, w)) + else: + if v is None: + raise ValueError("Query parameter {} cannot be None".format(k)) + query_params.append("{}={}".format(k, v)) + query = "?" + "&".join(query_params) + url += query + return url + +def lookup_encoding(encoding): + # type: (str) -> bool + # including check for whether encoding is known taken from httpx + try: + codecs.lookup(encoding) + return True + except LookupError: + return False + +def parse_lines_from_text(text): + # largely taken from httpx's LineDecoder code + lines = [] + last_chunk_of_text = "" + while text: + text_length = len(text) + for idx in range(text_length): + curr_char = text[idx] + next_char = None if idx == len(text) - 1 else text[idx + 1] + if curr_char == "\n": + lines.append(text[: idx + 1]) + text = text[idx + 1: ] + break + if curr_char == "\r" and next_char == "\n": + # if it ends with \r\n, we only do \n + lines.append(text[:idx] + "\n") + text = text[idx + 2:] + break + if curr_char == "\r" and next_char is not None: + # if it's \r then a normal character, we switch \r to \n + lines.append(text[:idx] + "\n") + text = text[idx + 1:] + break + if next_char is None: + last_chunk_of_text += text + text = "" + break + if last_chunk_of_text.endswith("\r"): + # if ends with \r, we switch \r to \n + lines.append(last_chunk_of_text[:-1] + "\n") + elif last_chunk_of_text: + lines.append(last_chunk_of_text) + return lines + +def to_pipeline_transport_request_helper(rest_request): + from ..pipeline.transport import HttpRequest as PipelineTransportHttpRequest + return PipelineTransportHttpRequest( + method=rest_request.method, + url=rest_request.url, + headers=rest_request.headers, + files=rest_request._files, # pylint: disable=protected-access + data=rest_request._data # pylint: disable=protected-access + ) + +def from_pipeline_transport_request_helper(request_class, pipeline_transport_request): + return request_class( + method=pipeline_transport_request.method, + url=pipeline_transport_request.url, + headers=pipeline_transport_request.headers, + files=pipeline_transport_request.files, + data=pipeline_transport_request.data + ) + +def from_pipeline_transport_response(response_class, pipeline_transport_response): + response = response_class( + request=pipeline_transport_response.request._from_pipeline_transport_request(), + internal_response=pipeline_transport_response.internal_response, + ) + response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access + return response diff --git a/sdk/core/azure-core/azure/core/rest/_helpers_py3.py b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py new file mode 100644 index 000000000000..310e5467d959 --- /dev/null +++ b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py @@ -0,0 +1,47 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import collections.abc +from typing import AsyncIterable, Dict, Iterable, Tuple, Union + +from six import Iterator +from ._helpers import ( + _shared_set_content_body, + HeadersType +) +ContentType = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] + +def set_content_body(content: ContentType) -> Tuple[ + HeadersType, ContentType +]: + headers, body = _shared_set_content_body(content) + if body is not None: + return headers, body + if isinstance(content, collections.abc.AsyncIterable): + return {}, content + raise TypeError( + "Unexpected type for 'content': '{}'. ".format(type(content)) + + "We expect 'content' to either be str, bytes, or an Iterable / AsyncIterable" + ) diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py new file mode 100644 index 000000000000..2743284de3dd --- /dev/null +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -0,0 +1,386 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import copy +import cgi +from json import loads + +from typing import TYPE_CHECKING, cast + +from azure.core.exceptions import HttpResponseError + +from .._utils import _case_insensitive_dict +from ..pipeline.transport import HttpRequest as PipelineTransportHttpRequest +from ._helpers import ( + FilesType, + lookup_encoding, + parse_lines_from_text, + set_content_body, + set_json_body, + set_multipart_body, + set_urlencoded_body, + format_parameters, + to_pipeline_transport_request_helper, + from_pipeline_transport_request_helper, +) +from ..exceptions import ResponseNotReadError +if TYPE_CHECKING: + from typing import ( + Iterable, + Optional, + Any, + Iterator, + Union, + Dict, + ) + from ._helpers import HeadersType + ByteStream = Iterable[bytes] + ContentType = Union[str, bytes, ByteStream] + + from ._helpers import HeadersType, ContentTypeBase as ContentType + + + +################################## CLASSES ###################################### + +class HttpRequest(object): + """Represents an HTTP request. + + :param method: HTTP method (GET, HEAD, etc.) + :type method: str or ~azure.core.protocol.HttpVerbs + :param str url: The url for your request + :keyword params: Query parameters to be mapped into your URL. Your input + should be a mapping or sequence of query name to query value(s). + :paramtype params: mapping or sequence + :keyword headers: HTTP headers you want in your request. Your input should + be a mapping or sequence of header name to header value. + :paramtype headers: mapping or sequence + :keyword dict data: Form data you want in your request body. Use for form-encoded data, i.e. + HTML forms. + :keyword any json: A JSON serializable object. We handle JSON-serialization for your + object, so use this for more complicated data structures than `data`. + :keyword files: Files you want to in your request body. Use for uploading files with + multipart encoding. Your input should be a mapping or sequence of file name to file content. + Use the `data` kwarg in addition if you want to include non-file data files as part of your request. + :paramtype files: mapping or sequence + :keyword content: Content you want in your request body. Think of it as the kwarg you should input + if your data doesn't fit into `json`, `data`, or `files`. Accepts a bytes type, or a generator + that yields bytes. + :paramtype content: str or bytes or iterable[bytes] or asynciterable[bytes] + :ivar str url: The URL this request is against. + :ivar str method: The method type of this request. + :ivar headers: The HTTP headers you passed in to your request + :vartype headers: mapping or sequence + :ivar bytes content: The content passed in for the request + """ + + def __init__(self, method, url, **kwargs): + # type: (str, str, Any) -> None + + self.url = url + self.method = method + + params = kwargs.pop("params", None) + if params: + self.url = format_parameters(self.url, params) + self._files = None + self._data = None + + default_headers = self._set_body( + content=kwargs.pop("content", None), + data=kwargs.pop("data", None), + files=kwargs.pop("files", None), + json=kwargs.pop("json", None), + ) + self.headers = _case_insensitive_dict(default_headers) + self.headers.update(kwargs.pop("headers", {})) + + if kwargs: + raise TypeError( + "You have passed in kwargs '{}' that are not valid kwargs.".format( + "', '".join(list(kwargs.keys())) + ) + ) + + def _set_body(self, content, data, files, json): + # type: (Optional[ContentType], Optional[dict], Optional[FilesType], Any) -> HeadersType + """Sets the body of the request, and returns the default headers + """ + default_headers = {} + if data is not None and not isinstance(data, dict): + # should we warn? + content = data + if content is not None: + default_headers, self._data = set_content_body(content) + return default_headers + if json is not None: + default_headers, self._data = set_json_body(json) + return default_headers + if files: + default_headers, self._files = set_multipart_body(files) + if data: + default_headers, self._data = set_urlencoded_body(data) + if files and data: + # little hacky, but for files we don't send a content type with + # boundary so requests / aiohttp etc deal with it + default_headers.pop("Content-Type") + return default_headers + + def _update_headers(self, default_headers): + # type: (Dict[str, str]) -> None + for name, value in default_headers.items(): + if name == "Transfer-Encoding" and "Content-Length" in self.headers: + continue + self.headers.setdefault(name, value) + + @property + def content(self): + # type: (...) -> Any + """Gets the request content. + """ + return self._data or self._files + + def __repr__(self): + # type: (...) -> str + return "".format( + self.method, self.url + ) + + def __deepcopy__(self, memo=None): + try: + return HttpRequest( + method=self.method, + url=self.url, + headers=self.headers, + files=copy.deepcopy(self._files), + data=copy.deepcopy(self._data), + ) + except (ValueError, TypeError): + return copy.copy(self) + + def _to_pipeline_transport_request(self): + return to_pipeline_transport_request_helper(self) + + @classmethod + def _from_pipeline_transport_request(cls, pipeline_transport_request): + return from_pipeline_transport_request_helper(cls, pipeline_transport_request) + +class _HttpResponseBase(object): # pylint: disable=too-many-instance-attributes + """Class for HttpResponse. + + :keyword request: The request that resulted in this response. + :paramtype request: ~azure.core.rest.HttpRequest + :ivar int status_code: The status code of this response + :ivar headers: The response headers + :vartype headers: dict[str, any] + :ivar str reason: The reason phrase for this response + :ivar bytes content: The response content in bytes + :ivar str url: The URL that resulted in this response + :ivar str encoding: The response encoding. Is settable, by default + is the response Content-Type header + :ivar str text: The response body as a string. + :ivar request: The request that resulted in this response. + :vartype request: ~azure.core.rest.HttpRequest + :ivar str content_type: The content type of the response + :ivar bool is_error: Whether this response is an error. + """ + + def __init__(self, **kwargs): + # type: (Any) -> None + self.request = kwargs.pop("request") + self.internal_response = kwargs.pop("internal_response") + self.status_code = None + self.headers = {} # type: HeadersType + self.reason = None + self.is_closed = False + self.is_stream_consumed = False + self._num_bytes_downloaded = 0 + self.content_type = None + self._json = None # this is filled in ContentDecodePolicy, when we deserialize + self._connection_data_block_size = None + self._content = None + + @property + def url(self): + # type: (...) -> str + """Returns the URL that resulted in this response""" + return self.request.url + + def _get_charset_encoding(self): + content_type = self.headers.get("Content-Type") + + if not content_type: + return None + _, params = cgi.parse_header(content_type) + encoding = params.get('charset') # -> utf-8 + if encoding is None or not lookup_encoding(encoding): + return None + return encoding + + def _get_content(self): + """Return the internal response's content""" + return self._content + + def _set_content(self, val): + """Set the internal response's content""" + self._content = val + + def _has_content(self): + """How to check if your internal response has content""" + return self._content is not None + + @property + def encoding(self): + # type: (...) -> Optional[str] + """Returns the response encoding. By default, is specified + by the response Content-Type header. + """ + + try: + return self._encoding + except AttributeError: + return self._get_charset_encoding() + + @encoding.setter + def encoding(self, value): + # type: (str) -> None + """Sets the response encoding""" + self._encoding = value + + @property + def text(self): + # type: (...) -> str + """Returns the response body as a string""" + encoding = self.encoding + if encoding == "utf-8" or encoding is None: + encoding = "utf-8-sig" + return self.content.decode(encoding) + + @property + def num_bytes_downloaded(self): + # type: (...) -> int + """See how many bytes of your stream response have been downloaded""" + return self._num_bytes_downloaded + + def json(self): + # type: (...) -> Any + """Returns the whole body as a json object. + + :return: The JSON deserialized response body + :rtype: any + :raises json.decoder.JSONDecodeError or ValueError (in python 2.7) if object is not JSON decodable: + """ + if not self._has_content(): + raise ResponseNotReadError() + if not self._json: + self._json = loads(self.text) + return self._json + + def raise_for_status(self): + # type: (...) -> None + """Raises an HttpResponseError if the response has an error status code. + + If response is good, does nothing. + """ + if cast(int, self.status_code) >= 400: + raise HttpResponseError(response=self) + + @property + def content(self): + # type: (...) -> bytes + """Return the response's content in bytes.""" + if not self._has_content(): + raise ResponseNotReadError() + return cast(bytes, self._get_content()) + + def __repr__(self): + # type: (...) -> str + content_type_str = ( + ", Content-Type: {}".format(self.content_type) if self.content_type else "" + ) + return "".format( + self.status_code, self.reason, content_type_str + ) + + @classmethod + def _from_pipeline_transport_response(cls, pipeline_transport_response): + return from_pipeline_transport_request_helper(cls, pipeline_transport_response) + +class HttpResponse(_HttpResponseBase): # pylint: disable=too-many-instance-attributes + + def __enter__(self): + # type: (...) -> HttpResponse + return self + + def close(self): + # type: (...) -> None + self.is_closed = True + self.internal_response.close() + + def __exit__(self, *args): + # type: (...) -> None + self.close() + + def read(self): + # type: (...) -> bytes + """ + Read the response's bytes. + + """ + if not self._has_content(): + self._set_content(b"".join(self.iter_bytes())) + return self.content + + def iter_raw(self, chunk_size=None): + # type: (Optional[int]) -> Iterator[bytes] + """Iterate over the raw response bytes + """ + raise NotImplementedError() + + def iter_bytes(self, chunk_size=None): + # type: (Optional[int]) -> Iterator[bytes] + """Iterate over the response bytes + """ + raise NotImplementedError() + + def iter_text(self, chunk_size=None): + # type: (int) -> Iterator[str] + """Iterate over the response text + """ + for byte in self.iter_bytes(chunk_size): + text = byte.decode(self.encoding or "utf-8") + yield text + + def iter_lines(self, chunk_size=None): + # type: (int) -> Iterator[str] + for text in self.iter_text(chunk_size): + lines = parse_lines_from_text(text) + for line in lines: + yield line + + def _close_stream(self): + # type: (...) -> None + self.is_stream_consumed = True + self.close() diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py new file mode 100644 index 000000000000..dd8eca7b7f4c --- /dev/null +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -0,0 +1,501 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import copy +import asyncio +import cgi +import collections +import collections.abc +from json import loads +from typing import ( + Any, + AsyncIterable, + AsyncIterator, + Dict, + Iterable, Iterator, + Optional, + Type, + Union, +) + +from azure.core.exceptions import HttpResponseError + +from .._utils import _case_insensitive_dict + +from ._helpers import ( + ParamsType, + FilesType, + HeadersType, + cast, + lookup_encoding, + parse_lines_from_text, + set_json_body, + set_multipart_body, + set_urlencoded_body, + format_parameters, + to_pipeline_transport_request_helper, + from_pipeline_transport_request_helper, +) +from ._helpers_py3 import set_content_body +from ..exceptions import ResponseNotReadError +from ..pipeline._tools import to_pipeline_transport_helper + +ContentType = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] + +class _AsyncContextManager(collections.abc.Awaitable): + + def __init__(self, wrapped: collections.abc.Awaitable): + super().__init__() + self.wrapped = wrapped + self.response = None + + def __await__(self): + return self.wrapped.__await__() + + async def __aenter__(self): + self.response = await self + return self.response + + async def __aexit__(self, *args): + await self.response.__aexit__(*args) + + async def close(self): + await self.response.close() + +################################## CLASSES ###################################### + +class HttpRequest: + """Represents an HTTP request. + + :param method: HTTP method (GET, HEAD, etc.) + :type method: str or ~azure.core.protocol.HttpVerbs + :param str url: The url for your request + :keyword params: Query parameters to be mapped into your URL. Your input + should be a mapping or sequence of query name to query value(s). + :paramtype params: mapping or sequence + :keyword headers: HTTP headers you want in your request. Your input should + be a mapping or sequence of header name to header value. + :paramtype headers: mapping or sequence + :keyword any json: A JSON serializable object. We handle JSON-serialization for your + object, so use this for more complicated data structures than `data`. + :keyword content: Content you want in your request body. Think of it as the kwarg you should input + if your data doesn't fit into `json`, `data`, or `files`. Accepts a bytes type, or a generator + that yields bytes. + :paramtype content: str or bytes or iterable[bytes] or asynciterable[bytes] + :keyword dict data: Form data you want in your request body. Use for form-encoded data, i.e. + HTML forms. + :keyword files: Files you want to in your request body. Use for uploading files with + multipart encoding. Your input should be a mapping or sequence of file name to file content. + Use the `data` kwarg in addition if you want to include non-file data files as part of your request. + :paramtype files: mapping or sequence + :ivar str url: The URL this request is against. + :ivar str method: The method type of this request. + :ivar headers: The HTTP headers you passed in to your request + :vartype headers: mapping or sequence + :ivar bytes content: The content passed in for the request + """ + + def __init__( + self, + method: str, + url: str, + *, + params: Optional[ParamsType] = None, + headers: Optional[HeadersType] = None, + json: Any = None, + content: Optional[ContentType] = None, + data: Optional[dict] = None, + files: Optional[FilesType] = None, + **kwargs + ): + self.url = url + self.method = method + + if params: + self.url = format_parameters(self.url, params) + self._files = None + self._data = None # type: Any + + default_headers = self._set_body( + content=content, + data=data, + files=files, + json=json, + ) + self.headers = _case_insensitive_dict(default_headers) + self.headers.update(headers or {}) + + if kwargs: + raise TypeError( + "You have passed in kwargs '{}' that are not valid kwargs.".format( + "', '".join(list(kwargs.keys())) + ) + ) + + def _set_body( + self, + content: Optional[ContentType], + data: Optional[dict], + files: Optional[FilesType], + json: Any, + ) -> HeadersType: + """Sets the body of the request, and returns the default headers + """ + default_headers = {} # type: HeadersType + if data is not None and not isinstance(data, dict): + # should we warn? + content = data + if content is not None: + default_headers, self._data = set_content_body(content) + return default_headers + if json is not None: + default_headers, self._data = set_json_body(json) + return default_headers + if files: + default_headers, self._files = set_multipart_body(files) + if data: + default_headers, self._data = set_urlencoded_body(data) + if files and data: + # little hacky, but for files we don't send a content type with + # boundary so requests / aiohttp etc deal with it + default_headers.pop("Content-Type") + return default_headers + + @property + def content(self) -> Any: + """Gets the request content. + """ + return self._data or self._files + + def __repr__(self) -> str: + return "".format( + self.method, self.url + ) + + def __deepcopy__(self, memo=None) -> "HttpRequest": + try: + return HttpRequest( + method=self.method, + url=self.url, + headers=self.headers, + files=copy.deepcopy(self._files), + data=copy.deepcopy(self._data), + ) + except (ValueError, TypeError): + return copy.copy(self) + + def _to_pipeline_transport_request(self): + return to_pipeline_transport_request_helper(self) + + @classmethod + def _from_pipeline_transport_request(cls, pipeline_transport_request): + return from_pipeline_transport_request_helper(cls, pipeline_transport_request) + +class _HttpResponseBase: # pylint: disable=too-many-instance-attributes + """Base class for HttpResponse and AsyncHttpResponse. + + :keyword request: The request that resulted in this response. + :paramtype request: ~azure.core.rest.HttpRequest + :ivar int status_code: The status code of this response + :ivar headers: The response headers + :vartype headers: dict[str, any] + :ivar str reason: The reason phrase for this response + :ivar bytes content: The response content in bytes + :ivar str url: The URL that resulted in this response + :ivar str encoding: The response encoding. Is settable, by default + is the response Content-Type header + :ivar str text: The response body as a string. + :ivar request: The request that resulted in this response. + :vartype request: ~azure.core.rest.HttpRequest + :ivar str content_type: The content type of the response + :ivar bool is_closed: Whether the network connection has been closed yet + :ivar bool is_stream_consumed: When getting a stream response, checks + whether the stream has been fully consumed + :ivar int num_bytes_downloaded: The number of bytes in your stream that + have been downloaded + """ + + def __init__( + self, + *, + request: HttpRequest, + internal_response, + **kwargs # pylint: disable=unused-argument + ): + self.request = request + self.internal_response = internal_response + self.status_code = None + self.headers = {} # type: HeadersType + self.reason = None + self.is_closed = False + self.is_stream_consumed = False + self._num_bytes_downloaded = 0 + self.content_type = None + self._connection_data_block_size = None + self._json = None # this is filled in ContentDecodePolicy, when we deserialize + self._content = None + + @property + def url(self) -> str: + """Returns the URL that resulted in this response""" + return self.request.url + + def _get_charset_encoding(self) -> Optional[str]: + content_type = self.headers.get("Content-Type") + + if not content_type: + return None + _, params = cgi.parse_header(content_type) + encoding = params.get('charset') # -> utf-8 + if encoding is None or not lookup_encoding(encoding): + return None + return encoding + + def _get_content(self): + """Return the internal response's content""" + return self._content + + def _set_content(self, val): + """Set the internal response's content""" + self._content = val + + def _has_content(self): + """How to check if your internal response has content""" + return self._content is not None + + @property + def encoding(self) -> Optional[str]: + """Returns the response encoding. By default, is specified + by the response Content-Type header. + """ + try: + return self._encoding + except AttributeError: + return self._get_charset_encoding() + + @encoding.setter + def encoding(self, value: str) -> None: + """Sets the response encoding""" + self._encoding = value + + @property + def text(self) -> str: + """Returns the response body as a string""" + encoding = self.encoding + if encoding == "utf-8" or encoding is None: + encoding = "utf-8-sig" + return self.content.decode(encoding) + + @property + def num_bytes_downloaded(self) -> int: + """See how many bytes of your stream response have been downloaded""" + return self._num_bytes_downloaded + + def json(self) -> Any: + """Returns the whole body as a json object. + + :return: The JSON deserialized response body + :rtype: any + :raises json.decoder.JSONDecodeError or ValueError (in python 2.7) if object is not JSON decodable: + """ + if not self._has_content(): + raise ResponseNotReadError() + if not self._json: + self._json = loads(self.text) + return self._json + + def raise_for_status(self) -> None: + """Raises an HttpResponseError if the response has an error status code. + + If response is good, does nothing. + """ + if cast(int, self.status_code) >= 400: + raise HttpResponseError(response=self) + + @property + def content(self) -> bytes: + """Return the response's content in bytes.""" + if not self._has_content(): + raise ResponseNotReadError() + return cast(bytes, self._get_content()) + + @classmethod + def _from_pipeline_transport_response(cls, pipeline_transport_response): + return from_pipeline_transport_request_helper(cls, pipeline_transport_response) + +class HttpResponse(_HttpResponseBase): + + def __enter__(self) -> "HttpResponse": + return self + + def close(self) -> None: + """Close the response + + :return: None + :rtype: None + """ + self.is_closed = True + self.internal_response.close() + + def __exit__(self, *args) -> None: + self.is_closed = True + self.internal_response.__exit__(*args) + + def read(self) -> bytes: + """Read the response's bytes. + + :return: The read in bytes + :rtype: bytes + """ + if not self._has_content(): + self._set_content(b"".join(self.iter_bytes())) + return self.content + + def iter_raw(self, chunk_size: Optional[int] = None) -> Iterator[bytes]: + """Iterates over the response's bytes. Will not decompress in the process + + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An iterator of bytes from the response + :rtype: Iterator[str] + """ + raise NotImplementedError() + + def iter_bytes(self, chunk_size: Optional[int] = None) -> Iterator[bytes]: + """Iterates over the response's bytes. Will decompress in the process + + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An iterator of bytes from the response + :rtype: Iterator[str] + """ + raise NotImplementedError() + + def iter_text(self, chunk_size: int = None) -> Iterator[str]: + """Iterates over the text in the response. + + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An iterator of string. Each string chunk will be a text from the response + :rtype: Iterator[str] + """ + for byte in self.iter_bytes(chunk_size): + text = byte.decode(self.encoding or "utf-8") + yield text + + def iter_lines(self, chunk_size: int = None) -> Iterator[str]: + """Iterates over the lines in the response. + + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An iterator of string. Each string chunk will be a line from the response + :rtype: Iterator[str] + """ + for text in self.iter_text(chunk_size): + lines = parse_lines_from_text(text) + for line in lines: + yield line + + def __repr__(self) -> str: + content_type_str = ( + ", Content-Type: {}".format(self.content_type) if self.content_type else "" + ) + return "".format( + self.status_code, self.reason, content_type_str + ) + +class AsyncHttpResponse(_HttpResponseBase): + + async def read(self) -> bytes: + """Read the response's bytes into memory. + + :return: The response's bytes + :rtype: bytes + """ + if not self._has_content(): + parts = [] + async for part in self.iter_bytes(): # type: ignore + parts.append(part) + self._set_content(b"".join(parts)) + return self._get_content() + + async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: + """Asynchronously iterates over the response's bytes. Will not decompress in the process + + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + raise NotImplementedError() + + async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: + """Asynchronously iterates over the response's bytes. Will decompress in the process + + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + raise NotImplementedError() + + async def iter_text(self, chunk_size: int = None) -> AsyncIterator[str]: + """Asynchronously iterates over the text in the response. + + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of string. Each string chunk will be a text from the response + :rtype: AsyncIterator[str] + """ + async for byte in self.iter_bytes(chunk_size): # type: ignore + text = byte.decode(self.encoding or "utf-8") + yield text + + async def iter_lines(self, chunk_size: int = None) -> AsyncIterator[str]: + """Asynchronously iterates over the lines in the response. + + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of string. Each string chunk will be a line from the response + :rtype: AsyncIterator[str] + """ + async for text in self.iter_text(chunk_size): + lines = parse_lines_from_text(text) + for line in lines: + yield line + + async def close(self) -> None: + """Close the response. + + :return: None + :rtype: None + """ + self.is_closed = True + self.internal_response.close() + await asyncio.sleep(0) + + async def __aexit__(self, *args) -> None: + self.is_closed = True + await self.internal_response.__aexit__(*args) + + def __repr__(self) -> str: + content_type_str = ( + ", Content-Type: {}".format(self.content_type) if self.content_type else "" + ) + return "".format( + self.status_code, self.reason, content_type_str + ) From cf637efb246454818b6e9923f6f2d148e0fbda95 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 19:04:55 -0400 Subject: [PATCH 28/64] add rest transport responses --- .../azure-core/azure/core/_pipeline_client.py | 26 ++++++ .../azure/core/_pipeline_client_async.py | 20 ++++ .../azure-core/azure/core/pipeline/_tools.py | 56 ++++++++++- .../azure/core/pipeline/_tools_async.py | 57 ++++++++++++ .../azure/core/pipeline/transport/_aiohttp.py | 93 +++++++++++++++++++ .../azure/core/pipeline/transport/_base.py | 12 +++ .../pipeline/transport/_requests_asyncio.py | 45 ++++++++- .../pipeline/transport/_requests_basic.py | 83 ++++++++++++++++- .../core/pipeline/transport/_requests_trio.py | 49 +++++++++- .../azure-core/azure/core/rest/_helpers.py | 8 -- sdk/core/azure-core/azure/core/rest/_rest.py | 5 - .../azure-core/azure/core/rest/_rest_py3.py | 5 - 12 files changed, 435 insertions(+), 24 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client.py b/sdk/core/azure-core/azure/core/_pipeline_client.py index 6f2376d36956..0e5a1615b05e 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client.py @@ -40,6 +40,7 @@ RetryPolicy, ) from .pipeline.transport import RequestsTransport +from .rest import HttpResponse try: from typing import TYPE_CHECKING @@ -59,6 +60,7 @@ Iterator, cast, ) # pylint: disable=unused-import + from .rest import HttpRequest _LOGGER = logging.getLogger(__name__) @@ -170,3 +172,27 @@ def _build_pipeline(self, config, **kwargs): # pylint: disable=no-self-use transport = RequestsTransport(**kwargs) return Pipeline(transport, policies) + + def send_request(self, request, **kwargs): + # type: (HttpRequest, Any) -> HttpResponse + """Runs the network request through the client's chained policies. + :param request: The network request you want to make. Required. + :type request: ~azure.core.rest.HttpRequest + :keyword bool stream: Whether the response payload will be streamed. Defaults to False. + :return: The response of your network call. Does not do error handling on your response. + :rtype: ~azure.core.rest.HttpResponse + # """ + rest_request = False + try: + request_to_run = request._to_pipeline_transport_request() + rest_request = True + except AttributeError: + request_to_run = request + return_pipeline_response = kwargs.pop("_return_pipeline_response", False) + pipeline_response = self._pipeline.run(request_to_run, **kwargs) # pylint: disable=protected-access + if return_pipeline_response: + return pipeline_response + response = pipeline_response.http_response + if rest_request: + return response._to_rest_response() + return response diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index 1e17480ba4f9..bc4ebabcd436 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -42,6 +42,8 @@ except ImportError: TYPE_CHECKING = False +from .rest import HttpRequest, _AsyncContextManager, AsyncHttpResponse + if TYPE_CHECKING: from typing import ( List, @@ -168,3 +170,21 @@ def _build_pipeline(self, config, **kwargs): # pylint: disable=no-self-use transport = AioHttpTransport(**kwargs) return AsyncPipeline(transport, policies) + + async def _make_pipeline_call(self, request, stream, **kwargs): + rest_request = False + try: + request_to_run = request._to_pipeline_transport_request() + rest_request = True + except AttributeError: + request_to_run = request + return_pipeline_response = kwargs.pop("_return_pipeline_response", False) + pipeline_response = await self._pipeline.run( + request_to_run, stream=stream, **kwargs # pylint: disable=protected-access + ) + if return_pipeline_response: + return pipeline_response + response = pipeline_response.http_response + if rest_request: + return response._to_rest_response() + return response diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools.py b/sdk/core/azure-core/azure/core/pipeline/_tools.py index 47453ad55721..c846f26aefb5 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- - +from ..rest import HttpRequest def await_result(func, *args, **kwargs): """If func returns an awaitable, raise that this runner can't handle it.""" result = func(*args, **kwargs) @@ -32,3 +32,57 @@ def await_result(func, *args, **kwargs): "Policy {} returned awaitable object in non-async pipeline.".format(func) ) return result + +def _stream_download_helper(decompress, stream_download_generator, response, chunk_size=None): + # type: (bool, Callable, HttpResponse, Optional[int]) -> Iterator[bytes] + if response.is_stream_consumed: + raise StreamConsumedError() + if response.is_closed: + raise StreamClosedError() + + response.is_stream_consumed = True + stream_download = stream_download_generator( + pipeline=None, + response=response, + chunk_size=chunk_size or response._connection_data_block_size, # pylint: disable=protected-access + decompress=decompress, + ) + for part in stream_download: + response._num_bytes_downloaded += len(part) + yield part + +def iter_bytes_helper(stream_download_generator, response, chunk_size=None): + # type: (Callable, HttpResponse, Optional[int]) -> Iterator[bytes] + if response._has_content(): # pylint: disable=protected-access + if chunk_size is None: + chunk_size = len(response.content) + for i in range(0, len(response.content), chunk_size): + yield response.content[i: i + chunk_size] + else: + for part in _stream_download_helper( + decompress=True, + stream_download_generator=stream_download_generator, + response=response, + chunk_size=chunk_size + ): + yield part + response.close() + +def iter_raw_helper(stream_download_generator, response, chunk_size=None): + # type: (Callable, HttpResponse, Optional[int]) -> Iterator[bytes] + for raw_bytes in _stream_download_helper( + decompress=False, + stream_download_generator=stream_download_generator, + response=response, + chunk_size=chunk_size + ): + yield raw_bytes + response.close() + +def to_rest_response_helper(pipeline_transport_response, response_type): + response = response_type( + request=pipeline_transport_response.request._to_rest_request(), # pylint: disable=protected-access + internal_response=pipeline_transport_response.internal_response, + ) + response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access + return response diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index d29988bd41ee..9144e16871a3 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -23,6 +23,8 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +from typing import Optional, Callable, AsyncIterator, TYPE_CHECKING, Union +from ..exceptions import StreamClosedError, StreamConsumedError async def await_result(func, *args, **kwargs): """If func returns an awaitable, await it.""" @@ -31,3 +33,58 @@ async def await_result(func, *args, **kwargs): # type ignore on await: https://github.com/python/mypy/issues/7587 return await result # type: ignore return result + +async def _stream_download_helper( + decompress: bool, + stream_download_generator: Callable, + response, + chunk_size: Optional[int] = None, +) -> AsyncIterator[bytes]: + if response.is_stream_consumed: + raise StreamConsumedError() + if response.is_closed: + raise StreamClosedError() + + response.is_stream_consumed = True + stream_download = stream_download_generator( + pipeline=None, + response=response, + chunk_size=chunk_size or response._connection_data_block_size, # pylint: disable=protected-access + decompress=decompress, + ) + async for part in stream_download: + response._num_bytes_downloaded += len(part) + yield part + +async def iter_bytes_helper( + stream_download_generator: Callable, + response, + chunk_size: Optional[int] = None, +) -> AsyncIterator[bytes]: + content = response._get_content() # pylint: disable=protected-access + if content is not None: + if chunk_size is None: + chunk_size = len(content) + for i in range(0, len(content), chunk_size): + yield content[i: i + chunk_size] + else: + async for raw_bytes in _stream_download_helper( + decompress=True, + stream_download_generator=stream_download_generator, + response=response, + chunk_size=chunk_size + ): + yield raw_bytes + +async def iter_raw_helper( + stream_download_generator: Callable, + response, + chunk_size: Optional[int] = None +) -> AsyncIterator[bytes]: + async for raw_bytes in _stream_download_helper( + decompress=False, + stream_download_generator=stream_download_generator, + response=response, + chunk_size=chunk_size + ): + yield raw_bytes diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 9ae3f96434cb..1ae5c2bdfe8a 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -46,6 +46,15 @@ AsyncHttpTransport, AsyncHttpResponse, _ResponseStopIteration) +from ...rest import ( + HttpRequest as RestHttpRequest, + AsyncHttpResponse as RestAsyncHttpResponse, +) +from .._tools import to_rest_response_helper +from .._tools_async import ( + iter_bytes_helper, + iter_raw_helper, +) # Matching requests, because why not? CONTENT_CHUNK_SIZE = 10 * 1024 @@ -358,3 +367,87 @@ def __getstate__(self): state['internal_response'] = None # aiohttp response are not pickable (see headers comments) state['headers'] = CIMultiDict(self.headers) # MultiDictProxy is not pickable return state + + def _to_rest_response(self): + return to_rest_response_helper(self, RestAioHttpTransportResponse) + +class RestAioHttpTransportResponse(RestAsyncHttpResponse): + def __init__( + self, + *, + request: RestHttpRequest, + internal_response, + **kwargs + ): + super().__init__(request=request, internal_response=internal_response, **kwargs) + self.status_code = internal_response.status + self.headers = CIMultiDict(internal_response.headers) # type: ignore + self.reason = internal_response.reason + self.content_type = internal_response.headers.get('content-type') + self._decompress = True + + @property + def text(self) -> str: + content = self.content + encoding = self.encoding + ctype = self.headers.get(aiohttp.hdrs.CONTENT_TYPE, "").lower() + mimetype = aiohttp.helpers.parse_mimetype(ctype) + + encoding = mimetype.parameters.get("charset") + if encoding: + try: + codecs.lookup(encoding) + except LookupError: + encoding = None + if not encoding: + if mimetype.type == "application" and ( + mimetype.subtype == "json" or mimetype.subtype == "rdap" + ): + # RFC 7159 states that the default encoding is UTF-8. + # RFC 7483 defines application/rdap+json + encoding = "utf-8" + elif content is None: + raise RuntimeError( + "Cannot guess the encoding of a not yet read content" + ) + else: + encoding = chardet.detect(content)["encoding"] + if not encoding: + encoding = "utf-8-sig" + + return content.decode(encoding) + + async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + """Asynchronously iterates over the response's bytes. Will not decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_raw_helper( + stream_download_generator=AioHttpStreamDownloadGenerator, + response=self, + chunk_size=chunk_size, + ): + yield part + await self.close() + + async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + """Asynchronously iterates over the response's bytes. Will decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_bytes_helper( + stream_download_generator=AioHttpStreamDownloadGenerator, + response=self, + chunk_size=chunk_size, + ): + yield part + await self.close() + + def __getstate__(self): + state = self.__dict__.copy() + # Remove the unpicklable entries. + state['internal_response'] = None # aiohttp response are not pickable (see headers comments) + state['headers'] = CIMultiDict(self.headers) # MultiDictProxy is not pickable + return state diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_base.py b/sdk/core/azure-core/azure/core/pipeline/transport/_base.py index 589d5549c584..31a9fa8a2d27 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_base.py @@ -473,6 +473,15 @@ def serialize(self): """ return _serialize_request(self) + def _to_rest_request(self): + from ...rest import HttpRequest as RestHttpRequest + return RestHttpRequest( + method=self.method, + url=self.url, + headers=self.headers, + files=self.files, + data=self.data + ) class _HttpResponseBase(object): """Represent a HTTP response. @@ -578,6 +587,9 @@ def __repr__(self): type(self).__name__, self.status_code, self.reason, content_type_str ) + def _to_rest_response(self): + """Convert PipelineTransport response to a Rest response""" + class HttpResponse(_HttpResponseBase): # pylint: disable=abstract-method def stream_download(self, pipeline, **kwargs): diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index aab184cb3d8b..6848627c94c6 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -44,8 +44,14 @@ _iterate_response_content) from ._requests_basic import RequestsTransportResponse, _read_raw_stream from ._base_requests_async import RequestsAsyncTransportBase - - +from .._tools import to_rest_response_helper +from .._tools_async import ( + iter_bytes_helper, + iter_raw_helper +) +from ...rest import ( + AsyncHttpResponse as RestAsyncHttpResponse, +) _LOGGER = logging.getLogger(__name__) @@ -186,3 +192,38 @@ class AsyncioRequestsTransportResponse(AsyncHttpResponse, RequestsTransportRespo def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]: # type: ignore """Generator for streaming request body data.""" return AsyncioStreamDownloadGenerator(pipeline, self, **kwargs) # type: ignore + + def _to_rest_response(self): + return to_rest_response_helper(self, RestAsyncioRequestsTransportResponse) + +class RestAsyncioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore + """Asynchronous streaming of data from the response. + """ + + async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + """Asynchronously iterates over the response's bytes. Will not decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_raw_helper( + stream_download_generator=AsyncioStreamDownloadGenerator, + response=self, + chunk_size=chunk_size, + ): + yield part + await self.close() + + async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + """Asynchronously iterates over the response's bytes. Will decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_bytes_helper( + stream_download_generator=AsyncioStreamDownloadGenerator, + response=self, + chunk_size=chunk_size, + ): + yield part + await self.close() diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py index b1b827424cdd..1721de98dd89 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py @@ -43,9 +43,18 @@ from ._base import ( HttpTransport, HttpResponse, - _HttpResponseBase + _HttpResponseBase, +) +from ...rest import ( + _HttpResponseBase as _RestHttpResponseBase, + HttpResponse as RestHttpResponse, ) from ._bigger_block_size_http_adapters import BiggerBlockSizeHTTPAdapter +from .._tools import ( + to_rest_response_helper, + iter_bytes_helper, + iter_raw_helper, +) PipelineType = TypeVar("PipelineType") @@ -160,6 +169,58 @@ def __next__(self): raise next = __next__ # Python 2 compatibility. +class _RestRequestsTransportResponseBase(_RestHttpResponseBase): + def __init__(self, **kwargs): + super(_RestRequestsTransportResponseBase, self).__init__(**kwargs) + self.status_code = self.internal_response.status_code + self.headers = self.internal_response.headers + self.reason = self.internal_response.reason + self.content_type = self.internal_response.headers.get('content-type') + + def _get_content(self): + """Return the internal response's content""" + if not self.internal_response._content_consumed: # pylint: disable=protected-access + # if we just call .content, requests will read in the content. + # we want to read it in our own way + return None + try: + return self.internal_response.content + except RuntimeError: + # requests throws a RuntimeError if the content for a response is already consumed + return None + + def _set_content(self, val): + """Set the internal response's content""" + self.internal_response._content = val # pylint: disable=protected-access + + def _has_content(self): + return self._get_content() is not None + + @_RestHttpResponseBase.encoding.setter # type: ignore + def encoding(self, value): + # type: (str) -> None + # ignoring setter bc of known mypy issue https://github.com/python/mypy/issues/1465 + self._encoding = value + encoding = value + if not encoding: + # There is a few situation where "requests" magic doesn't fit us: + # - https://github.com/psf/requests/issues/654 + # - https://github.com/psf/requests/issues/1737 + # - https://github.com/psf/requests/issues/2086 + from codecs import BOM_UTF8 + if self.internal_response.content[:3] == BOM_UTF8: + encoding = "utf-8-sig" + if encoding: + if encoding == "utf-8": + encoding = "utf-8-sig" + self.internal_response.encoding = encoding + + @property + def text(self): + if not self._has_content(): + raise ResponseNotReadError() + return self.internal_response.text + class RequestsTransportResponse(HttpResponse, _RequestsTransportResponseBase): """Streaming of data from the response. @@ -169,6 +230,26 @@ def stream_download(self, pipeline, **kwargs): """Generator for streaming request body data.""" return StreamDownloadGenerator(pipeline, self, **kwargs) + def _to_rest_response(self): + return to_rest_response_helper(self, RestRequestsTransportResponse) + +class RestRequestsTransportResponse(RestHttpResponse, _RestRequestsTransportResponseBase): + + def iter_bytes(self, chunk_size=None): + # type: (Optional[int]) -> Iterator[bytes] + return iter_bytes_helper( + stream_download_generator=StreamDownloadGenerator, + response=self, + chunk_size=chunk_size, + ) + + def iter_raw(self, chunk_size=None): + # type: (Optional[int]) -> Iterator[bytes] + return iter_raw_helper( + stream_download_generator=StreamDownloadGenerator, + response=self, + chunk_size=chunk_size, + ) class RequestsTransport(HttpTransport): """Implements a basic requests HTTP sender. diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index 7be76336979f..44440e5b2101 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -44,8 +44,14 @@ _iterate_response_content) from ._requests_basic import RequestsTransportResponse, _read_raw_stream from ._base_requests_async import RequestsAsyncTransportBase - - +from .._tools import to_rest_response_helper +from .._tools_async import ( + iter_raw_helper, + iter_bytes_helper, +) +from ...rest import ( + AsyncHttpResponse as RestAsyncHttpResponse, +) _LOGGER = logging.getLogger(__name__) @@ -107,6 +113,45 @@ def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]: # ty """ return TrioStreamDownloadGenerator(pipeline, self, **kwargs) + def _to_rest_response(self): + return to_rest_response_helper(self, RestTrioRequestsTransportResponse) + +class RestTrioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore + """Asynchronous streaming of data from the response. + """ + async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + """Asynchronously iterates over the response's bytes. Will not decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_raw_helper( + stream_download_generator=TrioStreamDownloadGenerator, + response=self, + chunk_size=chunk_size, + ): + yield part + await self.close() + + async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + """Asynchronously iterates over the response's bytes. Will decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_bytes_helper( + stream_download_generator=TrioStreamDownloadGenerator, + response=self, + chunk_size=chunk_size, + ): + yield part + await self.close() + + async def close(self) -> None: + self.is_closed = True + self.internal_response.close() + await trio.sleep(0) + class TrioRequestsTransport(RequestsAsyncTransportBase): # type: ignore """Identical implementation as the synchronous RequestsTransport wrapped in a class with diff --git a/sdk/core/azure-core/azure/core/rest/_helpers.py b/sdk/core/azure-core/azure/core/rest/_helpers.py index 61b41d14e9c0..2fcd8e83e7a2 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers.py @@ -273,11 +273,3 @@ def from_pipeline_transport_request_helper(request_class, pipeline_transport_req files=pipeline_transport_request.files, data=pipeline_transport_request.data ) - -def from_pipeline_transport_response(response_class, pipeline_transport_response): - response = response_class( - request=pipeline_transport_response.request._from_pipeline_transport_request(), - internal_response=pipeline_transport_response.internal_response, - ) - response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access - return response diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index 2743284de3dd..e4ab3d6a2068 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -32,7 +32,6 @@ from azure.core.exceptions import HttpResponseError from .._utils import _case_insensitive_dict -from ..pipeline.transport import HttpRequest as PipelineTransportHttpRequest from ._helpers import ( FilesType, lookup_encoding, @@ -324,10 +323,6 @@ def __repr__(self): self.status_code, self.reason, content_type_str ) - @classmethod - def _from_pipeline_transport_response(cls, pipeline_transport_response): - return from_pipeline_transport_request_helper(cls, pipeline_transport_response) - class HttpResponse(_HttpResponseBase): # pylint: disable=too-many-instance-attributes def __enter__(self): diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index dd8eca7b7f4c..9500a24f7f44 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -60,7 +60,6 @@ ) from ._helpers_py3 import set_content_body from ..exceptions import ResponseNotReadError -from ..pipeline._tools import to_pipeline_transport_helper ContentType = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] @@ -341,10 +340,6 @@ def content(self) -> bytes: raise ResponseNotReadError() return cast(bytes, self._get_content()) - @classmethod - def _from_pipeline_transport_response(cls, pipeline_transport_response): - return from_pipeline_transport_request_helper(cls, pipeline_transport_response) - class HttpResponse(_HttpResponseBase): def __enter__(self) -> "HttpResponse": From 0e596ce1e61d87958ff22d15d6d9c6946e53129a Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 19:05:55 -0400 Subject: [PATCH 29/64] add rest resposne mishandling exceptiosn to exceptions.py --- sdk/core/azure-core/azure/core/exceptions.py | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/sdk/core/azure-core/azure/core/exceptions.py b/sdk/core/azure-core/azure/core/exceptions.py index 4af83e9b3683..ddeec110d59f 100644 --- a/sdk/core/azure-core/azure/core/exceptions.py +++ b/sdk/core/azure-core/azure/core/exceptions.py @@ -433,3 +433,27 @@ def __str__(self): if self._error_format: return str(self._error_format) return super(ODataV4Error, self).__str__() + +class StreamConsumedError(Exception): + def __init__(self): + message = ( + "You are attempting to read or stream content that has already been streamed. " + "You have likely already consumed this stream, so it can not be accessed anymore." + ) + super(StreamConsumedError, self).__init__(message) + +class StreamClosedError(Exception): + def __init__(self): + message = ( + "You can not try to read or stream this response's content, since the " + "response has been closed." + ) + super(StreamClosedError, self).__init__(message) + +class ResponseNotReadError(Exception): + + def __init__(self): + message = ( + "You have not read in the response's bytes yet. Call response.read() first." + ) + super(ResponseNotReadError, self).__init__(message) From d20fa00380d8d9200351402c00cdab0c47a2ae05 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 19:08:31 -0400 Subject: [PATCH 30/64] current tests passing --- .../azure/core/pipeline/transport/_requests_asyncio.py | 2 +- .../azure-core/azure/core/pipeline/transport/_requests_trio.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index 6848627c94c6..610d9e4c3596 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -42,7 +42,7 @@ AsyncHttpResponse, _ResponseStopIteration, _iterate_response_content) -from ._requests_basic import RequestsTransportResponse, _read_raw_stream +from ._requests_basic import RequestsTransportResponse, _read_raw_stream, _RestRequestsTransportResponseBase from ._base_requests_async import RequestsAsyncTransportBase from .._tools import to_rest_response_helper from .._tools_async import ( diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index 44440e5b2101..5fdbe4100fbb 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -42,7 +42,7 @@ AsyncHttpResponse, _ResponseStopIteration, _iterate_response_content) -from ._requests_basic import RequestsTransportResponse, _read_raw_stream +from ._requests_basic import RequestsTransportResponse, _read_raw_stream, _RestRequestsTransportResponseBase from ._base_requests_async import RequestsAsyncTransportBase from .._tools import to_rest_response_helper from .._tools_async import ( From 883c7615a3bca5043e2e4f4340a345e202ac12c7 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 19:18:40 -0400 Subject: [PATCH 31/64] adding initial tests --- .../tests/testserver_tests/conftest.py | 5 ++ .../tests/testserver_tests/rest_client.py | 83 +++++++++++++++++++ .../test_rest_context_manager.py | 80 ++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 sdk/core/azure-core/tests/testserver_tests/rest_client.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 10a99fb3ce21..8cb5e80e6d6f 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -31,6 +31,7 @@ import sys import random from six.moves import urllib +from rest_client import TestRestClient def is_port_available(port_num): req = urllib.request.Request("http://localhost:{}/health".format(port_num)) @@ -86,3 +87,7 @@ def testserver(): collect_ignore_glob = [] if sys.version_info < (3, 5): collect_ignore_glob.append("*_async.py") + +@pytest.fixture +def client(port): + return TestRestClient(port) diff --git a/sdk/core/azure-core/tests/testserver_tests/rest_client.py b/sdk/core/azure-core/tests/testserver_tests/rest_client.py new file mode 100644 index 000000000000..d5d5bee8ddeb --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/rest_client.py @@ -0,0 +1,83 @@ + +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from azure.core.pipeline import policies +from azure.core.configuration import Configuration +from azure.core import PipelineClient +from copy import deepcopy + + +class TestRestClientConfiguration(Configuration): + def __init__( + self, **kwargs + ): + # type: (...) -> None + super(TestRestClientConfiguration, self).__init__(**kwargs) + + kwargs.setdefault("sdk_moniker", "autorestswaggerbatfileservice/1.0.0b1") + self._configure(**kwargs) + + def _configure( + self, **kwargs + ): + # type: (...) -> None + self.user_agent_policy = kwargs.get("user_agent_policy") or policies.UserAgentPolicy(**kwargs) + self.headers_policy = kwargs.get("headers_policy") or policies.HeadersPolicy(**kwargs) + self.proxy_policy = kwargs.get("proxy_policy") or policies.ProxyPolicy(**kwargs) + self.logging_policy = kwargs.get("logging_policy") or policies.NetworkTraceLoggingPolicy(**kwargs) + self.http_logging_policy = kwargs.get("http_logging_policy") or policies.HttpLoggingPolicy(**kwargs) + self.retry_policy = kwargs.get("retry_policy") or policies.RetryPolicy(**kwargs) + self.custom_hook_policy = kwargs.get("custom_hook_policy") or policies.CustomHookPolicy(**kwargs) + self.redirect_policy = kwargs.get("redirect_policy") or policies.RedirectPolicy(**kwargs) + self.authentication_policy = kwargs.get("authentication_policy") + +class TestRestClient(object): + + def __init__(self, port, **kwargs): + self._config = TestRestClientConfiguration(**kwargs) + self._client = PipelineClient( + base_url="http://localhost:{}/".format(port), + config=self._config, + **kwargs + ) + + def send_request(self, request, **kwargs): + """Runs the network request through the client's chained policies. + >>> from azure.core.rest import HttpRequest + >>> request = HttpRequest("GET", "http://localhost:3000/helloWorld") + + >>> response = client.send_request(request) + + For more information on this code flow, see https://aka.ms/azsdk/python/protocol/quickstart + :param request: The network request you want to make. Required. + :type request: ~azure.core.rest.HttpRequest + :keyword bool stream: Whether the response payload will be streamed. Defaults to False. + :return: The response of your network call. Does not do error handling on your response. + :rtype: ~azure.core.rest.HttpResponse + """ + request_copy = deepcopy(request) + request_copy.url = self._client.format_url(request_copy.url) + return self._client.send_request(request, **kwargs) \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py new file mode 100644 index 000000000000..488ea715ef49 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py @@ -0,0 +1,80 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +import pytest +import mock +from azure.core.rest import HttpRequest +from azure.core.exceptions import HttpResponseError, ResponseNotReadError + +from azure.core.pipeline import Pipeline, transport +from azure.core.pipeline.transport import RequestsTransport + +def test_normal_call(client, port): + def _raise_and_get_text(response): + response.raise_for_status() + assert response.text == "Hello, world!" + assert response.is_closed + request = HttpRequest("GET", url="/basic/string") + response = client.send_request(request) + _raise_and_get_text(response) + assert response.is_closed + + with client.send_request(request) as response: + _raise_and_get_text(response) + + response = client.send_request(request) + with response as response: + _raise_and_get_text(response) + +def test_stream_call(client): + def _raise_and_get_text(response): + response.raise_for_status() + assert not response.is_closed + with pytest.raises(ResponseNotReadError): + response.text + response.read() + assert response.text == "Hello, world!" + assert response.is_closed + request = HttpRequest("GET", url="/streams/basic") + response = client.send_request(request, stream=True) + _raise_and_get_text(response) + assert response.is_closed + + with client.send_request(request, stream=True) as response: + _raise_and_get_text(response) + assert response.is_closed + + response = client.send_request(request, stream=True) + with response as response: + _raise_and_get_text(response) + +def test_stream_with_error(client): + request = HttpRequest("GET", url="/streams/error") + with client.send_request(request, stream=True) as response: + assert not response.is_closed + with pytest.raises(HttpResponseError) as e: + response.raise_for_status() + error = e.value + assert error.status_code == 400 + assert error.reason == "BAD REQUEST" + assert "Operation returned an invalid status 'BAD REQUEST'" in str(error) + with pytest.raises(ResponseNotReadError): + error.error + with pytest.raises(ResponseNotReadError): + error.model + with pytest.raises(ResponseNotReadError): + response.json() + with pytest.raises(ResponseNotReadError): + response.content + + # NOW WE READ THE RESPONSE + response.read() + assert error.status_code == 400 + assert error.reason == "BAD REQUEST" + assert error.error.code == "BadRequest" + assert error.error.message == "You made a bad request" + assert error.model.code == "BadRequest" + assert error.error.message == "You made a bad request" From e35d198b6abcb4c6ee2ba0a719e5d66d70aea34b Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 11:33:11 -0400 Subject: [PATCH 32/64] most sync tests passing, working on async --- .../azure-core/azure/core/_pipeline_client.py | 28 +- .../azure/core/_pipeline_client_async.py | 37 ++- .../azure-core/azure/core/pipeline/_tools.py | 11 +- .../azure/core/pipeline/_tools_async.py | 2 +- .../azure/core/pipeline/transport/_aiohttp.py | 6 +- .../pipeline/transport/_requests_basic.py | 8 +- .../tests/testserver_tests/rest_client.py | 2 +- .../testserver_tests/rest_client_async.py | 69 ++++ .../test_rest_context_manager.py | 54 ++-- .../test_rest_context_manager_async.py | 84 +++++ .../testserver_tests/test_rest_headers.py | 104 ++++++ .../test_rest_http_request.py | 306 ++++++++++++++++++ .../test_rest_http_response.py | 304 +++++++++++++++++ .../tests/testserver_tests/test_rest_query.py | 31 ++ .../test_rest_stream_responses.py | 260 +++++++++++++++ 15 files changed, 1253 insertions(+), 53 deletions(-) create mode 100644 sdk/core/azure-core/tests/testserver_tests/rest_client_async.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager_async.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/test_rest_headers.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/test_rest_query.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py diff --git a/sdk/core/azure-core/azure/core/_pipeline_client.py b/sdk/core/azure-core/azure/core/_pipeline_client.py index 0e5a1615b05e..0ddb224b91e2 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client.py @@ -64,6 +64,16 @@ _LOGGER = logging.getLogger(__name__) +def _prepare_request(request): + # returns the request ready to run through pipelines + # and a bool telling whether we ended up converting it + rest_request = False + try: + request_to_run = request._to_pipeline_transport_request() + rest_request = True + except AttributeError: + request_to_run = request + return rest_request, request_to_run class PipelineClient(PipelineClientBase): """Service client core methods. @@ -173,6 +183,7 @@ def _build_pipeline(self, config, **kwargs): # pylint: disable=no-self-use return Pipeline(transport, policies) + def send_request(self, request, **kwargs): # type: (HttpRequest, Any) -> HttpResponse """Runs the network request through the client's chained policies. @@ -182,17 +193,16 @@ def send_request(self, request, **kwargs): :return: The response of your network call. Does not do error handling on your response. :rtype: ~azure.core.rest.HttpResponse # """ - rest_request = False - try: - request_to_run = request._to_pipeline_transport_request() - rest_request = True - except AttributeError: - request_to_run = request + rest_request, request_to_run = _prepare_request(request) return_pipeline_response = kwargs.pop("_return_pipeline_response", False) pipeline_response = self._pipeline.run(request_to_run, **kwargs) # pylint: disable=protected-access - if return_pipeline_response: - return pipeline_response response = pipeline_response.http_response if rest_request: - return response._to_rest_response() + response = response._to_rest_response() + if not kwargs.get("stream", False): + response.read() + response.close() + if return_pipeline_response: + pipeline_response.http_response = response + return pipeline_response return response diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index bc4ebabcd436..6b556272acc7 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -36,7 +36,8 @@ RequestIdPolicy, AsyncRetryPolicy, ) - +from ._pipeline_client import _prepare_request +from typing import Any, Awaitable try: from typing import TYPE_CHECKING except ImportError: @@ -172,19 +173,35 @@ def _build_pipeline(self, config, **kwargs): # pylint: disable=no-self-use return AsyncPipeline(transport, policies) async def _make_pipeline_call(self, request, stream, **kwargs): - rest_request = False - try: - request_to_run = request._to_pipeline_transport_request() - rest_request = True - except AttributeError: - request_to_run = request + rest_request, request_to_run = _prepare_request(request) return_pipeline_response = kwargs.pop("_return_pipeline_response", False) pipeline_response = await self._pipeline.run( request_to_run, stream=stream, **kwargs # pylint: disable=protected-access ) - if return_pipeline_response: - return pipeline_response response = pipeline_response.http_response if rest_request: - return response._to_rest_response() + response = response._to_rest_response() + if not kwargs.get("stream", False): + await response.read() + await response.close() + if return_pipeline_response: + pipeline_response.http_response = response + return pipeline_response return response + + def send_request( + self, + request: HttpRequest, + *, + stream: bool = False, + **kwargs: Any + ) -> Awaitable[AsyncHttpResponse]: + """Runs the network request through the client's chained policies. + :param request: The network request you want to make. Required. + :type request: ~azure.core.rest.HttpRequest + :keyword bool stream: Whether the response payload will be streamed. Defaults to False. + :return: The response of your network call. Does not do error handling on your response. + :rtype: ~azure.core.rest.AsyncHttpResponse + """ + wrapped = self._make_pipeline_call(request, stream=stream, **kwargs) + return _AsyncContextManager(wrapped=wrapped) \ No newline at end of file diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools.py b/sdk/core/azure-core/azure/core/pipeline/_tools.py index c846f26aefb5..125d65b9e314 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from ..rest import HttpRequest +from ..exceptions import StreamClosedError, StreamConsumedError def await_result(func, *args, **kwargs): """If func returns an awaitable, raise that this runner can't handle it.""" result = func(*args, **kwargs) @@ -86,3 +86,12 @@ def to_rest_response_helper(pipeline_transport_response, response_type): ) response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access return response + +def get_chunk_size(response, **kwargs): + chunk_size = kwargs.pop("chunk_size", None) + if not chunk_size: + if hasattr(response, "block_size"): + chunk_size = response.block_size + elif hasattr(response, "_connection_data_block_size"): + chunk_size = response._connection_data_block_size + return chunk_size diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index 9144e16871a3..a6ddfb6f2a98 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -49,7 +49,7 @@ async def _stream_download_helper( stream_download = stream_download_generator( pipeline=None, response=response, - chunk_size=chunk_size or response._connection_data_block_size, # pylint: disable=protected-access + chunk_size=chunk_size, decompress=decompress, ) async for part in stream_download: diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 1ae5c2bdfe8a..0c6963177619 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -50,7 +50,7 @@ HttpRequest as RestHttpRequest, AsyncHttpResponse as RestAsyncHttpResponse, ) -from .._tools import to_rest_response_helper +from .._tools import to_rest_response_helper, get_chunk_size from .._tools_async import ( iter_bytes_helper, iter_raw_helper, @@ -220,11 +220,11 @@ class AioHttpStreamDownloadGenerator(AsyncIterator): :param bool decompress: If True which is default, will attempt to decode the body based on the *content-encoding* header. """ - def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, *, decompress=True) -> None: + def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, *, decompress=True, **kwargs) -> None: self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = response.block_size + self.block_size = get_chunk_size(response, **kwargs) self._decompress = decompress self.content_length = int(response.internal_response.headers.get('Content-Length', 0)) self._decompressor = None diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py index 1721de98dd89..6a167e3709de 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py @@ -36,7 +36,8 @@ from azure.core.configuration import ConnectionConfiguration from azure.core.exceptions import ( ServiceRequestError, - ServiceResponseError + ServiceResponseError, + ResponseNotReadError, ) from . import HttpRequest # pylint: disable=unused-import @@ -54,6 +55,7 @@ to_rest_response_helper, iter_bytes_helper, iter_raw_helper, + get_chunk_size, ) PipelineType = TypeVar("PipelineType") @@ -79,6 +81,8 @@ def _read_raw_stream(response, chunk_size=1): if not chunk: break yield chunk + # following behavior from requests iter_content, we set content consumed to True + response._content_consumed = True # pylint: disable=protected-access class _RequestsTransportResponseBase(_HttpResponseBase): """Base class for accessing response data. @@ -136,7 +140,7 @@ def __init__(self, pipeline, response, **kwargs): self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = response.block_size + self.block_size = get_chunk_size(response, **kwargs) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) diff --git a/sdk/core/azure-core/tests/testserver_tests/rest_client.py b/sdk/core/azure-core/tests/testserver_tests/rest_client.py index d5d5bee8ddeb..2d896ac3b6aa 100644 --- a/sdk/core/azure-core/tests/testserver_tests/rest_client.py +++ b/sdk/core/azure-core/tests/testserver_tests/rest_client.py @@ -80,4 +80,4 @@ def send_request(self, request, **kwargs): """ request_copy = deepcopy(request) request_copy.url = self._client.format_url(request_copy.url) - return self._client.send_request(request, **kwargs) \ No newline at end of file + return self._client.send_request(request_copy, **kwargs) \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/rest_client_async.py b/sdk/core/azure-core/tests/testserver_tests/rest_client_async.py new file mode 100644 index 000000000000..1f2e3568bb02 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/rest_client_async.py @@ -0,0 +1,69 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from copy import deepcopy +from azure.core import AsyncPipelineClient +from azure.core.pipeline import policies +from azure.core.configuration import Configuration + +class TestRestClientConfiguration(Configuration): + def __init__( + self, **kwargs + ): + # type: (...) -> None + super(TestRestClientConfiguration, self).__init__(**kwargs) + + kwargs.setdefault("sdk_moniker", "autorestswaggerbatfileservice/1.0.0b1") + self._configure(**kwargs) + + def _configure(self, **kwargs) -> None: + self.user_agent_policy = kwargs.get("user_agent_policy") or policies.UserAgentPolicy(**kwargs) + self.headers_policy = kwargs.get("headers_policy") or policies.HeadersPolicy(**kwargs) + self.proxy_policy = kwargs.get("proxy_policy") or policies.ProxyPolicy(**kwargs) + self.logging_policy = kwargs.get("logging_policy") or policies.NetworkTraceLoggingPolicy(**kwargs) + self.http_logging_policy = kwargs.get("http_logging_policy") or policies.HttpLoggingPolicy(**kwargs) + self.retry_policy = kwargs.get("retry_policy") or policies.AsyncRetryPolicy(**kwargs) + self.custom_hook_policy = kwargs.get("custom_hook_policy") or policies.CustomHookPolicy(**kwargs) + self.redirect_policy = kwargs.get("redirect_policy") or policies.AsyncRedirectPolicy(**kwargs) + self.authentication_policy = kwargs.get("authentication_policy") + +class AsyncTestRestClient(object): + + def __init__(self, port, **kwargs): + self._config = TestRestClientConfiguration(**kwargs) + + self._client = AsyncPipelineClient( + base_url="http://localhost:{}".format(port), + config=self._config, + **kwargs + ) + + def send_request(self, request, **kwargs): + """Runs the network request through the client's chained policies. + >>> from azure.core.rest import HttpRequest + >>> request = HttpRequest("GET", "http://localhost:3000/helloWorld") + + >>> response = await client.send_request(request) + + For more information on this code flow, see https://aka.ms/azsdk/python/protocol/quickstart + :param request: The network request you want to make. Required. + :type request: ~azure.core.rest.HttpRequest + :keyword bool stream: Whether the response payload will be streamed. Defaults to False. + :return: The response of your network call. Does not do error handling on your response. + :rtype: ~azure.core.rest.AsyncHttpResponse + """ + request_copy = deepcopy(request) + request_copy.url = self._client.format_url(request_copy.url) + return self._client.send_request(request_copy, **kwargs) + + async def close(self) -> None: + await self._client.close() + + async def __aenter__(self): + await self._client.__aenter__() + return self + + async def __aexit__(self, *exc_details) -> None: + await self._client.__aexit__(*exc_details) \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py index 488ea715ef49..c705fe6045da 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py @@ -51,30 +51,32 @@ def _raise_and_get_text(response): with response as response: _raise_and_get_text(response) -def test_stream_with_error(client): - request = HttpRequest("GET", url="/streams/error") - with client.send_request(request, stream=True) as response: - assert not response.is_closed - with pytest.raises(HttpResponseError) as e: - response.raise_for_status() - error = e.value - assert error.status_code == 400 - assert error.reason == "BAD REQUEST" - assert "Operation returned an invalid status 'BAD REQUEST'" in str(error) - with pytest.raises(ResponseNotReadError): - error.error - with pytest.raises(ResponseNotReadError): - error.model - with pytest.raises(ResponseNotReadError): - response.json() - with pytest.raises(ResponseNotReadError): - response.content +# TODO: commenting until https://github.com/Azure/azure-sdk-for-python/issues/18086 is fixed - # NOW WE READ THE RESPONSE - response.read() - assert error.status_code == 400 - assert error.reason == "BAD REQUEST" - assert error.error.code == "BadRequest" - assert error.error.message == "You made a bad request" - assert error.model.code == "BadRequest" - assert error.error.message == "You made a bad request" +# def test_stream_with_error(client): +# request = HttpRequest("GET", url="/streams/error") +# with client.send_request(request, stream=True) as response: +# assert not response.is_closed +# with pytest.raises(HttpResponseError) as e: +# response.raise_for_status() +# error = e.value +# assert error.status_code == 400 +# assert error.reason == "BAD REQUEST" +# assert "Operation returned an invalid status 'BAD REQUEST'" in str(error) +# with pytest.raises(ResponseNotReadError): +# error.error +# with pytest.raises(ResponseNotReadError): +# error.model +# with pytest.raises(ResponseNotReadError): +# response.json() +# with pytest.raises(ResponseNotReadError): +# response.content + +# # NOW WE READ THE RESPONSE +# response.read() +# assert error.status_code == 400 +# assert error.reason == "BAD REQUEST" +# assert error.error.code == "BadRequest" +# assert error.error.message == "You made a bad request" +# assert error.model.code == "BadRequest" +# assert error.error.message == "You made a bad request" diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager_async.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager_async.py new file mode 100644 index 000000000000..9f92fc87a924 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager_async.py @@ -0,0 +1,84 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from azure.core.exceptions import HttpResponseError, ResponseNotReadError +import pytest +from azure.core.rest import HttpRequest +from rest_client_async import AsyncTestRestClient + +@pytest.fixture +def client(port): + return AsyncTestRestClient(port) + +@pytest.mark.asyncio +async def test_normal_call(client): + async def _raise_and_get_text(response): + response.raise_for_status() + assert response.text == "Hello, world!" + assert response.is_closed + request = HttpRequest("GET", url="/basic/string") + response = await client.send_request(request) + await _raise_and_get_text(response) + assert response.is_closed + + async with client.send_request(request) as response: + await _raise_and_get_text(response) + + response = client.send_request(request) + async with response as response: + await _raise_and_get_text(response) + +@pytest.mark.asyncio +async def test_stream_call(client): + async def _raise_and_get_text(response): + response.raise_for_status() + assert not response.is_closed + with pytest.raises(ResponseNotReadError): + response.text + await response.read() + assert response.text == "Hello, world!" + assert response.is_closed + request = HttpRequest("GET", url="/streams/basic") + response = await client.send_request(request, stream=True) + await _raise_and_get_text(response) + assert response.is_closed + + async with client.send_request(request, stream=True) as response: + await _raise_and_get_text(response) + assert response.is_closed + + response = client.send_request(request, stream=True) + async with response as response: + await _raise_and_get_text(response) + +@pytest.mark.asyncio +async def test_stream_with_error(client): + request = HttpRequest("GET", url="/streams/error") + async with client.send_request(request, stream=True) as response: + assert not response.is_closed + with pytest.raises(HttpResponseError) as e: + response.raise_for_status() + error = e.value + assert error.status_code == 400 + assert error.reason == "BAD REQUEST" + assert "Operation returned an invalid status 'BAD REQUEST'" in str(error) + with pytest.raises(ResponseNotReadError): + error.error + with pytest.raises(ResponseNotReadError): + error.model + with pytest.raises(ResponseNotReadError): + response.json() + with pytest.raises(ResponseNotReadError): + response.content + + # NOW WE READ THE RESPONSE + await response.read() + assert error.status_code == 400 + assert error.reason == "BAD REQUEST" + assert error.error.code == "BadRequest" + assert error.error.message == "You made a bad request" + assert error.model.code == "BadRequest" + assert error.error.message == "You made a bad request" \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_headers.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_headers.py new file mode 100644 index 000000000000..30112c50c912 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_headers.py @@ -0,0 +1,104 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +import sys + +# NOTE: These tests are heavily inspired from the httpx test suite: https://github.com/encode/httpx/tree/master/tests +# Thank you httpx for your wonderful tests! +from azure.core.rest import HttpRequest + +def _get_headers(header_value): + request = HttpRequest(method="GET", url="http://example.org", headers=header_value) + return request.headers + +def test_headers(): + # headers still can't be list of tuples. Will uncomment once we add this support + # h = _get_headers([("a", "123"), ("a", "456"), ("b", "789")]) + # assert "a" in h + # assert "A" in h + # assert "b" in h + # assert "B" in h + # assert "c" not in h + # assert h["a"] == "123, 456" + # assert h.get("a") == "123, 456" + # assert h.get("nope", default=None) is None + # assert h.get_list("a") == ["123", "456"] + + # assert list(h.keys()) == ["a", "b"] + # assert list(h.values()) == ["123, 456", "789"] + # assert list(h.items()) == [("a", "123, 456"), ("b", "789")] + # assert list(h) == ["a", "b"] + # assert dict(h) == {"a": "123, 456", "b": "789"} + # assert repr(h) == "Headers([('a', '123'), ('a', '456'), ('b', '789')])" + # assert h == [("a", "123"), ("b", "789"), ("a", "456")] + # assert h == [("a", "123"), ("A", "456"), ("b", "789")] + # assert h == {"a": "123", "A": "456", "b": "789"} + # assert h != "a: 123\nA: 456\nb: 789" + + h = _get_headers({"a": "123", "b": "789"}) + assert h["A"] == "123" + assert h["B"] == "789" + + +def test_header_mutations(): + h = _get_headers({}) + assert dict(h) == {} + h["a"] = "1" + assert dict(h) == {"a": "1"} + h["a"] = "2" + assert dict(h) == {"a": "2"} + h.setdefault("a", "3") + assert dict(h) == {"a": "2"} + h.setdefault("b", "4") + assert dict(h) == {"a": "2", "b": "4"} + del h["a"] + assert dict(h) == {"b": "4"} + + +def test_headers_insert_retains_ordering(): + h = _get_headers({"a": "a", "b": "b", "c": "c"}) + h["b"] = "123" + if sys.version_info >= (3, 6): + assert list(h.values()) == ["a", "123", "c"] + else: + assert set(list(h.values())) == set(["a", "123", "c"]) + + +def test_headers_insert_appends_if_new(): + h = _get_headers({"a": "a", "b": "b", "c": "c"}) + h["d"] = "123" + if sys.version_info >= (3, 6): + assert list(h.values()) == ["a", "b", "c", "123"] + else: + assert set(list(h.values())) == set(["a", "b", "c", "123"]) + + +def test_headers_insert_removes_all_existing(): + h = _get_headers([("a", "123"), ("a", "456")]) + h["a"] = "789" + assert dict(h) == {"a": "789"} + + +def test_headers_delete_removes_all_existing(): + h = _get_headers([("a", "123"), ("a", "456")]) + del h["a"] + assert dict(h) == {} + +def test_headers_not_override(): + request = HttpRequest("PUT", "http://example.org", json={"hello": "world"}, headers={"Content-Length": "5000", "Content-Type": "application/my-content-type"}) + assert request.headers["Content-Length"] == "5000" + assert request.headers["Content-Type"] == "application/my-content-type" + +# Can't support list of tuples. Will uncomment once we add that support + +# def test_multiple_headers(): +# """ +# `Headers.get_list` should support both split_commas=False and split_commas=True. +# """ +# h = _get_headers([("set-cookie", "a, b"), ("set-cookie", "c")]) +# assert h.get_list("Set-Cookie") == ["a, b", "c"] + +# h = _get_headers([("vary", "a, b"), ("vary", "c")]) +# assert h.get_list("Vary", split_commas=True) == ["a", "b", "c"] \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py new file mode 100644 index 000000000000..cde6e83de490 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py @@ -0,0 +1,306 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +# NOTE: These tests are heavily inspired from the httpx test suite: https://github.com/encode/httpx/tree/master/tests +# Thank you httpx for your wonderful tests! +import io +import pytest +import sys +import collections +from typing import Generator +from azure.core.rest import HttpRequest + +@pytest.fixture +def assert_iterator_body(): + def _comparer(request, final_value): + content = b"".join([p for p in request.content]) + assert content == final_value + return _comparer + +def test_request_repr(): + request = HttpRequest("GET", "http://example.org") + assert repr(request) == "" + +def test_no_content(): + request = HttpRequest("GET", "http://example.org") + assert "Content-Length" not in request.headers + +def test_content_length_header(): + request = HttpRequest("POST", "http://example.org", content=b"test 123") + assert request.headers["Content-Length"] == "8" + + +def test_iterable_content(assert_iterator_body): + class Content: + def __iter__(self): + yield b"test 123" # pragma: nocover + + request = HttpRequest("POST", "http://example.org", content=Content()) + assert request.headers == {} + assert_iterator_body(request, b"test 123") + + +def test_generator_with_transfer_encoding_header(assert_iterator_body): + def content(): + yield b"test 123" # pragma: nocover + + request = HttpRequest("POST", "http://example.org", content=content()) + assert request.headers == {} + assert_iterator_body(request, b"test 123") + + +def test_generator_with_content_length_header(assert_iterator_body): + def content(): + yield b"test 123" # pragma: nocover + + headers = {"Content-Length": "8"} + request = HttpRequest( + "POST", "http://example.org", content=content(), headers=headers + ) + assert request.headers == {"Content-Length": "8"} + assert_iterator_body(request, b"test 123") + + +def test_url_encoded_data(): + request = HttpRequest("POST", "http://example.org", data={"test": "123"}) + + assert request.headers["Content-Type"] == "application/x-www-form-urlencoded" + assert request.content == {'test': '123'} # httpx makes this just b'test=123'. set_formdata_body is still keeping it as a dict + + +def test_json_encoded_data(): + request = HttpRequest("POST", "http://example.org", json={"test": 123}) + + assert request.headers["Content-Type"] == "application/json" + assert request.content == '{"test": 123}' + + +def test_headers(): + request = HttpRequest("POST", "http://example.org", json={"test": 123}) + + assert request.headers == { + "Content-Type": "application/json", + "Content-Length": "13", + } + + +def test_ignore_transfer_encoding_header_if_content_length_exists(): + """ + `Transfer-Encoding` should be ignored if `Content-Length` has been set explicitly. + See https://github.com/encode/httpx/issues/1168 + """ + + def streaming_body(data): + yield data # pragma: nocover + + data = streaming_body(b"abcd") + + headers = {"Content-Length": "4"} + request = HttpRequest("POST", "http://example.org", data=data, headers=headers) + assert "Transfer-Encoding" not in request.headers + assert request.headers["Content-Length"] == "4" + +def test_override_accept_encoding_header(): + headers = {"Accept-Encoding": "identity"} + + request = HttpRequest("GET", "http://example.org", headers=headers) + assert request.headers["Accept-Encoding"] == "identity" + +"""Test request body""" +def test_empty_content(): + request = HttpRequest("GET", "http://example.org") + assert request.content is None + +def test_string_content(): + request = HttpRequest("PUT", "http://example.org", content="Hello, world!") + assert request.headers == {"Content-Length": "13", "Content-Type": "text/plain"} + assert request.content == "Hello, world!" + + # Support 'data' for compat with requests. + request = HttpRequest("PUT", "http://example.org", data="Hello, world!") + + assert request.headers == {"Content-Length": "13", "Content-Type": "text/plain"} + assert request.content == "Hello, world!" + + # content length should not be set for GET requests + + request = HttpRequest("GET", "http://example.org", data="Hello, world!") + + assert request.headers == {"Content-Length": "13", "Content-Type": "text/plain"} + assert request.content == "Hello, world!" + +@pytest.mark.skipif(sys.version_info < (3, 0), + reason="In 2.7, b'' is the same as a string, so will have text/plain content type") +def test_bytes_content(): + request = HttpRequest("PUT", "http://example.org", content=b"Hello, world!") + assert request.headers == {"Content-Length": "13"} + assert request.content == b"Hello, world!" + + # Support 'data' for compat with requests. + request = HttpRequest("PUT", "http://example.org", data=b"Hello, world!") + + assert request.headers == {"Content-Length": "13"} + assert request.content == b"Hello, world!" + + # should still be set regardless of method + + request = HttpRequest("GET", "http://example.org", data=b"Hello, world!") + + assert request.headers == {"Content-Length": "13"} + assert request.content == b"Hello, world!" + +def test_iterator_content(assert_iterator_body): + # NOTE: in httpx, content reads out the actual value. Don't do that (yet) in azure rest + def hello_world(): + yield b"Hello, " + yield b"world!" + + request = HttpRequest("POST", url="http://example.org", content=hello_world()) + assert isinstance(request.content, collections.Iterable) + + assert_iterator_body(request, b"Hello, world!") + assert request.headers == {} + + # Support 'data' for compat with requests. + request = HttpRequest("POST", url="http://example.org", data=hello_world()) + assert isinstance(request.content, collections.Iterable) + + assert_iterator_body(request, b"Hello, world!") + assert request.headers == {} + + # transfer encoding should still be set for GET requests + request = HttpRequest("GET", url="http://example.org", data=hello_world()) + assert isinstance(request.content, collections.Iterable) + + assert_iterator_body(request, b"Hello, world!") + assert request.headers == {} + + +def test_json_content(): + request = HttpRequest("POST", url="http://example.org", json={"Hello": "world!"}) + + assert request.headers == { + "Content-Length": "19", + "Content-Type": "application/json", + } + assert request.content == '{"Hello": "world!"}' + +def test_urlencoded_content(): + # NOTE: not adding content length setting and content testing bc we're not adding content length in the rest code + # that's dealt with later in the pipeline. + request = HttpRequest("POST", url="http://example.org", data={"Hello": "world!"}) + assert request.headers == { + "Content-Type": "application/x-www-form-urlencoded", + } + +@pytest.mark.parametrize(("key"), (1, 2.3, None)) +def test_multipart_invalid_key(key): + + data = {key: "abc"} + files = {"file": io.BytesIO(b"")} + with pytest.raises(TypeError) as e: + HttpRequest( + url="http://127.0.0.1:8000/", + method="POST", + data=data, + files=files, + ) + assert "Invalid type for data name" in str(e.value) + assert repr(key) in str(e.value) + + +@pytest.mark.skipif(sys.version_info < (3, 0), + reason="In 2.7, b'' is the same as a string, so check doesn't fail") +def test_multipart_invalid_key_binary_string(): + + data = {b"abc": "abc"} + files = {"file": io.BytesIO(b"")} + with pytest.raises(TypeError) as e: + HttpRequest( + url="http://127.0.0.1:8000/", + method="POST", + data=data, + files=files, + ) + assert "Invalid type for data name" in str(e.value) + assert repr(b"abc") in str(e.value) + +@pytest.mark.parametrize(("value"), (object(), {"key": "value"})) +def test_multipart_invalid_value(value): + + data = {"text": value} + files = {"file": io.BytesIO(b"")} + with pytest.raises(TypeError) as e: + HttpRequest("POST", "http://127.0.0.1:8000/", data=data, files=files) + assert "Invalid type for data value" in str(e.value) + +def test_empty_request(): + request = HttpRequest("POST", url="http://example.org", data={}, files={}) + + assert request.headers == {} + assert not request.content # in core, we don't convert urlencoded dict to bytes representation in content + +def test_read_content(assert_iterator_body): + def content(): + yield b"test 123" + + request = HttpRequest("POST", "http://example.org", content=content()) + assert_iterator_body(request, b"test 123") + # in this case, request._data is what we end up passing to the requests transport + assert isinstance(request._data, collections.Iterable) + + +def test_complicated_json(client): + # thanks to Sean Kane for this test! + input = { + 'EmptyByte': '', + 'EmptyUnicode': '', + 'SpacesOnlyByte': ' ', + 'SpacesOnlyUnicode': ' ', + 'SpacesBeforeByte': ' Text', + 'SpacesBeforeUnicode': ' Text', + 'SpacesAfterByte': 'Text ', + 'SpacesAfterUnicode': 'Text ', + 'SpacesBeforeAndAfterByte': ' Text ', + 'SpacesBeforeAndAfterUnicode': ' Text ', + '啊齄丂狛': 'ꀕ', + 'RowKey': 'test2', + '啊齄丂狛狜': 'hello', + "singlequote": "a''''b", + "doublequote": 'a""""b', + "None": None, + } + request = HttpRequest("POST", "http://localhost:5000/basic/complicated-json", json=input) + r = client.send_request(request) + r.raise_for_status() + +# NOTE: For files, we don't allow list of tuples yet, just dict. Will uncomment when we add this capability +# def test_multipart_multiple_files_single_input_content(): +# files = [ +# ("file", io.BytesIO(b"")), +# ("file", io.BytesIO(b"")), +# ] +# request = HttpRequest("POST", url="http://example.org", files=files) +# assert request.headers == { +# "Content-Length": "271", +# "Content-Type": "multipart/form-data; boundary=+++", +# } +# assert request.content == b"".join( +# [ +# b"--+++\r\n", +# b'Content-Disposition: form-data; name="file"; filename="upload"\r\n', +# b"Content-Type: application/octet-stream\r\n", +# b"\r\n", +# b"\r\n", +# b"--+++\r\n", +# b'Content-Disposition: form-data; name="file"; filename="upload"\r\n', +# b"Content-Type: application/octet-stream\r\n", +# b"\r\n", +# b"\r\n", +# b"--+++--\r\n", +# ] +# ) \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py new file mode 100644 index 000000000000..7214f088bca0 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py @@ -0,0 +1,304 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +# NOTE: These tests are heavily inspired from the httpx test suite: https://github.com/encode/httpx/tree/master/tests +# Thank you httpx for your wonderful tests! +import io +import sys +import pytest +from azure.core.rest import HttpRequest +from azure.core.exceptions import HttpResponseError +import xml.etree.ElementTree as ET + +@pytest.fixture +def send_request(client): + def _send_request(request): + response = client.send_request(request, stream=False) + response.raise_for_status() + return response + return _send_request + +def test_response(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/string"), + ) + assert response.status_code == 200 + assert response.reason == "OK" + assert response.text == "Hello, world!" + assert response.request.method == "GET" + assert response.request.url == "http://localhost:5000/basic/string" + + +def test_response_content(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/bytes"), + ) + assert response.status_code == 200 + assert response.reason == "OK" + assert response.text == "Hello, world!" + + +def test_response_text(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/string"), + ) + assert response.status_code == 200 + assert response.reason == "OK" + assert response.text == "Hello, world!" + assert response.headers["Content-Length"] == '13' + assert response.headers['Content-Type'] == "text/plain; charset=utf-8" + assert response.content_type == "text/plain; charset=utf-8" + +def test_response_html(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/html"), + ) + assert response.status_code == 200 + assert response.reason == "OK" + assert response.text == "Hello, world!" + +def test_raise_for_status(client): + response = client.send_request( + HttpRequest("GET", "http://localhost:5000/basic/string"), + ) + response.raise_for_status() + + response = client.send_request( + HttpRequest("GET", "http://localhost:5000/errors/403"), + ) + assert response.status_code == 403 + with pytest.raises(HttpResponseError): + response.raise_for_status() + + response = client.send_request( + HttpRequest("GET", "http://localhost:5000/errors/500"), + retry_total=0, # takes too long with retires on 500 + ) + assert response.status_code == 500 + with pytest.raises(HttpResponseError): + response.raise_for_status() + +def test_response_repr(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/string") + ) + assert repr(response) == "" + +def test_response_content_type_encoding(send_request): + """ + Use the charset encoding in the Content-Type header if possible. + """ + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1") + ) + assert response.content_type == "text/plain; charset=latin-1" + assert response.text == u"Latin 1: ÿ" + assert response.encoding == "latin-1" + + +def test_response_autodetect_encoding(send_request): + """ + Autodetect encoding if there is no Content-Type header. + """ + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1") + ) + + assert response.text == u'Latin 1: ÿ' + assert response.encoding == "latin-1" + +@pytest.mark.skipif(sys.version_info < (3, 0), + reason="In 2.7, b'' is the same as a string, so will have text/plain content type") +def test_response_fallback_to_autodetect(send_request): + """ + Fallback to autodetection if we get an invalid charset in the Content-Type header. + """ + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/invalid-codec-name") + ) + + assert response.headers["Content-Type"] == "text/plain; charset=invalid-codec-name" + assert response.text == u"おはようございます。" + assert response.encoding is None + + +def test_response_no_charset_with_ascii_content(send_request): + """ + A response with ascii encoded content should decode correctly, + even with no charset specified. + """ + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/no-charset"), + ) + + assert response.headers["Content-Type"] == "text/plain" + assert response.status_code == 200 + assert response.encoding is None + assert response.text == "Hello, world!" + + +def test_response_no_charset_with_iso_8859_1_content(send_request): + """ + A response with ISO 8859-1 encoded content should decode correctly, + even with no charset specified. + """ + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/iso-8859-1"), + ) + assert response.text == u"Accented: Österreich" + assert response.encoding is None + +def test_response_set_explicit_encoding(send_request): + # Deliberately incorrect charset + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1-with-utf-8"), + ) + assert response.headers["Content-Type"] == "text/plain; charset=utf-8" + response.encoding = "latin-1" + assert response.text == u"Latin 1: ÿ" + assert response.encoding == "latin-1" + +def test_json(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/json"), + ) + assert response.json() == {"greeting": "hello", "recipient": "world"} + assert response.encoding is None + +def test_json_with_specified_encoding(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/json"), + ) + assert response.json() == {"greeting": "hello", "recipient": "world"} + assert response.encoding == "utf-16" + +def test_emoji(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/emoji"), + ) + assert response.text == u"👩" + +def test_emoji_family_with_skin_tone_modifier(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/emoji-family-skin-tone-modifier"), + ) + assert response.text == u"👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987" + +def test_korean_nfc(send_request): + response = send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/korean"), + ) + assert response.text == u"아가" + +def test_urlencoded_content(send_request): + send_request( + request=HttpRequest( + "POST", + "http://localhost:5000/urlencoded/pet/add/1", + data={ "pet_type": "dog", "pet_food": "meat", "name": "Fido", "pet_age": 42 } + ), + ) + +def test_multipart_files_content(send_request): + request = HttpRequest( + "POST", + "http://localhost:5000/multipart/basic", + files={"fileContent": io.BytesIO(b"")}, + ) + send_request(request) + +def test_multipart_data_and_files_content(send_request): + request = HttpRequest( + "POST", + "http://localhost:5000/multipart/data-and-files", + data={"message": "Hello, world!"}, + files={"fileContent": io.BytesIO(b"")}, + ) + send_request(request) + +def test_multipart_encode_non_seekable_filelike(send_request): + """ + Test that special readable but non-seekable filelike objects are supported, + at the cost of reading them into memory at most once. + """ + + class IteratorIO(io.IOBase): + def __init__(self, iterator): + self._iterator = iterator + + def read(self, *args): + return b"".join(self._iterator) + + def data(): + yield b"Hello" + yield b"World" + + fileobj = IteratorIO(data()) + files = {"file": fileobj} + request = HttpRequest( + "POST", + "http://localhost:5000/multipart/non-seekable-filelike", + files=files, + ) + send_request(request) + +def test_get_xml_basic(send_request): + request = HttpRequest( + "GET", + "http://localhost:5000/xml/basic", + ) + response = send_request(request) + parsed_xml = ET.fromstring(response.text) + assert parsed_xml.tag == 'slideshow' + attributes = parsed_xml.attrib + assert attributes['title'] == "Sample Slide Show" + assert attributes['date'] == "Date of publication" + assert attributes['author'] == "Yours Truly" + +def test_put_xml_basic(send_request): + + basic_body = """ + + + Wake up to WonderWidgets! + + + Overview + Why WonderWidgets are great + + Who buys WonderWidgets + +""" + + request = HttpRequest( + "PUT", + "http://localhost:5000/xml/basic", + content=ET.fromstring(basic_body), + ) + send_request(request) + +class MockHttpRequest(HttpRequest): + """Use this to check how many times _convert() was called""" + def __init__(self, *args, **kwargs): + super(MockHttpRequest, self).__init__(*args, **kwargs) + self.num_calls_to_convert = 0 + + def _convert(self): + self.num_calls_to_convert += 1 + return super(MockHttpRequest, self)._convert() + + +def test_request_no_conversion(send_request): + request = MockHttpRequest("GET", "http://localhost:5000/basic/string") + response = send_request( + request=request, + ) + assert response.status_code == 200 + assert request.num_calls_to_convert == 0 \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_query.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_query.py new file mode 100644 index 000000000000..7933e998f1e6 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_query.py @@ -0,0 +1,31 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +# NOTE: These tests are heavily inspired from the httpx test suite: https://github.com/encode/httpx/tree/master/tests +# Thank you httpx for your wonderful tests! + +import pytest +from azure.core.rest import HttpRequest + +def _format_query_into_url(url, params): + request = HttpRequest(method="GET", url=url, params=params) + return request.url + +def test_request_url_with_params(): + url = _format_query_into_url(url="a/b/c?t=y", params={"g": "h"}) + assert url in ["a/b/c?g=h&t=y", "a/b/c?t=y&g=h"] + +def test_request_url_with_params_as_list(): + url = _format_query_into_url(url="a/b/c?t=y", params={"g": ["h","i"]}) + assert url in ["a/b/c?g=h&g=i&t=y", "a/b/c?t=y&g=h&g=i"] + +def test_request_url_with_params_with_none_in_list(): + with pytest.raises(ValueError): + _format_query_into_url(url="a/b/c?t=y", params={"g": ["h",None]}) + +def test_request_url_with_params_with_none(): + with pytest.raises(ValueError): + _format_query_into_url(url="a/b/c?t=y", params={"g": None}) \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py new file mode 100644 index 000000000000..c8eee614ea6f --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py @@ -0,0 +1,260 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +import pytest +from azure.core.rest import HttpRequest +from azure.core.exceptions import StreamClosedError, StreamConsumedError, ResponseNotReadError +from azure.core.exceptions import HttpResponseError, ServiceRequestError + +def _assert_stream_state(response, open): + # if open is true, check the stream is open. + # if false, check if everything is closed + checks = [ + response.internal_response._content_consumed, + response.is_closed, + response.is_stream_consumed + ] + if open: + assert not any(checks) + else: + assert all(checks) + +def test_iter_raw(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + with client.send_request(request, stream=True) as response: + raw = b"" + for part in response.iter_raw(): + assert not response.internal_response._content_consumed + assert not response.is_closed + assert response.is_stream_consumed # we follow httpx behavior here + raw += part + assert raw == b"Hello, world!" + assert response.internal_response._content_consumed + assert response.is_closed + assert response.is_stream_consumed + +def test_iter_raw_on_iterable(client): + request = HttpRequest("GET", "http://localhost:5000/streams/iterable") + + with client.send_request(request, stream=True) as response: + raw = b"" + for part in response.iter_raw(): + raw += part + assert raw == b"Hello, world!" + +def test_iter_with_error(client): + request = HttpRequest("GET", "http://localhost:5000/errors/403") + + with client.send_request(request, stream=True) as response: + with pytest.raises(HttpResponseError): + response.raise_for_status() + assert response.is_closed + + with pytest.raises(HttpResponseError): + with client.send_request(request, stream=True) as response: + response.raise_for_status() + assert response.is_closed + + request = HttpRequest("GET", "http://doesNotExist") + with pytest.raises(ServiceRequestError): + with client.send_request(request, stream=True) as response: + raise ValueError("Should error before entering") + assert response.is_closed + +def test_iter_raw_with_chunksize(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + with client.send_request(request, stream=True) as response: + parts = [part for part in response.iter_raw(chunk_size=5)] + assert parts == [b"Hello", b", wor", b"ld!"] + + with client.send_request(request, stream=True) as response: + parts = [part for part in response.iter_raw(chunk_size=13)] + assert parts == [b"Hello, world!"] + + with client.send_request(request, stream=True) as response: + parts = [part for part in response.iter_raw(chunk_size=20)] + assert parts == [b"Hello, world!"] + +def test_iter_raw_num_bytes_downloaded(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + with client.send_request(request, stream=True) as response: + num_downloaded = response.num_bytes_downloaded + for part in response.iter_raw(): + assert len(part) == (response.num_bytes_downloaded - num_downloaded) + num_downloaded = response.num_bytes_downloaded + +def test_iter_bytes(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + with client.send_request(request, stream=True) as response: + raw = b"" + for chunk in response.iter_bytes(): + assert not response.internal_response._content_consumed + assert not response.is_closed + assert response.is_stream_consumed # we follow httpx behavior here + raw += chunk + assert response.internal_response._content_consumed + assert response.is_closed + assert response.is_stream_consumed + assert raw == b"Hello, world!" + +def test_iter_bytes_with_chunk_size(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + with client.send_request(request, stream=True) as response: + parts = [part for part in response.iter_bytes(chunk_size=5)] + assert parts == [b"Hello", b", wor", b"ld!"] + + with client.send_request(request, stream=True) as response: + parts = [part for part in response.iter_bytes(chunk_size=13)] + assert parts == [b"Hello, world!"] + + with client.send_request(request, stream=True) as response: + parts = [part for part in response.iter_bytes(chunk_size=20)] + assert parts == [b"Hello, world!"] + +def test_iter_text(client): + request = HttpRequest("GET", "http://localhost:5000/basic/string") + + with client.send_request(request, stream=True) as response: + content = "" + for part in response.iter_text(): + content += part + assert content == "Hello, world!" + +def test_iter_text_with_chunk_size(client): + request = HttpRequest("GET", "http://localhost:5000/basic/string") + + with client.send_request(request, stream=True) as response: + parts = [part for part in response.iter_text(chunk_size=5)] + assert parts == ["Hello", ", wor", "ld!"] + + with client.send_request(request, stream=True) as response: + parts = [part for part in response.iter_text(chunk_size=13)] + assert parts == ["Hello, world!"] + + with client.send_request(request, stream=True) as response: + parts = [part for part in response.iter_text(chunk_size=20)] + assert parts == ["Hello, world!"] + +def test_iter_lines(client): + request = HttpRequest("GET", "http://localhost:5000/basic/lines") + + with client.send_request(request, stream=True) as response: + content = [] + for line in response.iter_lines(): + content.append(line) + assert content == ["Hello,\n", "world!"] + +def test_sync_streaming_response(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + with client.send_request(request, stream=True) as response: + assert response.status_code == 200 + assert not response.is_closed + + content = response.read() + + assert content == b"Hello, world!" + assert response.content == b"Hello, world!" + assert response.is_closed + +def test_cannot_read_after_stream_consumed(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + with client.send_request(request, stream=True) as response: + content = b"" + for part in response.iter_bytes(): + content += part + + assert content == b"Hello, world!" + + with pytest.raises(StreamConsumedError) as ex: + response.read() + + assert "You are attempting to read or stream content that has already been streamed." in str(ex.value) + +def test_cannot_read_after_response_closed(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + with client.send_request(request, stream=True) as response: + response.close() + with pytest.raises(StreamClosedError) as ex: + response.read() + assert "You can not try to read or stream this response's content, since the response has been closed" in str(ex.value) + +def test_decompress_plain_no_header(client): + # thanks to Xiang Yan for this test! + account_name = "coretests" + url = "https://{}.blob.core.windows.net/tests/test.txt".format(account_name) + request = HttpRequest("GET", url) + response = client.send_request(request, stream=True) + with pytest.raises(ResponseNotReadError): + response.content + response.read() + assert response.content == b"test" + +def test_compress_plain_no_header(client): + # thanks to Xiang Yan for this test! + account_name = "coretests" + url = "https://{}.blob.core.windows.net/tests/test.txt".format(account_name) + request = HttpRequest("GET", url) + response = client.send_request(request, stream=True) + iter = response.iter_raw() + data = b"".join(list(iter)) + assert data == b"test" + +def test_decompress_compressed_no_header(client): + # thanks to Xiang Yan for this test! + account_name = "coretests" + url = "https://{}.blob.core.windows.net/tests/test.tar.gz".format(account_name) + request = HttpRequest("GET", url) + response = client.send_request(request, stream=True) + iter = response.iter_bytes() + data = b"".join(list(iter)) + assert data == b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\n+I-.\x01\x00\x0c~\x7f\xd8\x04\x00\x00\x00' + +def test_decompress_compressed_header(client): + # thanks to Xiang Yan for this test! + account_name = "coretests" + account_url = "https://{}.blob.core.windows.net".format(account_name) + url = "https://{}.blob.core.windows.net/tests/test_with_header.tar.gz".format(account_name) + request = HttpRequest("GET", url) + response = client.send_request(request, stream=True) + iter = response.iter_text() + data = "".join(list(iter)) + assert data == "test" + +def test_iter_read(client): + # thanks to McCoy Patiño for this test! + request = HttpRequest("GET", "http://localhost:5000/basic/lines") + response = client.send_request(request, stream=True) + response.read() + iterator = response.iter_lines() + for line in iterator: + assert line + assert response.text + +def test_iter_read_back_and_forth(client): + # thanks to McCoy Patiño for this test! + + # while this test may look like it's exposing buggy behavior, this is httpx's behavior + # the reason why the code flow is like this, is because the 'iter_x' functions don't + # actually read the contents into the response, the output them. Once they're yielded, + # the stream is closed, so you have to catch the output when you iterate through it + request = HttpRequest("GET", "http://localhost:5000/basic/lines") + response = client.send_request(request, stream=True) + iterator = response.iter_lines() + for line in iterator: + assert line + with pytest.raises(ResponseNotReadError): + response.text + with pytest.raises(StreamConsumedError): + response.read() + with pytest.raises(ResponseNotReadError): + response.text \ No newline at end of file From b26a9e788c320f0359e3df5f25150325583c4b7a Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 12:02:17 -0400 Subject: [PATCH 33/64] most async tests passing --- .../azure/core/_pipeline_client_async.py | 12 +- .../azure/core/pipeline/_tools_async.py | 36 +-- .../azure/core/pipeline/transport/_aiohttp.py | 23 +- .../async_tests/testserver_tests/conftest.py | 93 ++++++ .../coretestserver/coretestserver/__init__.py | 33 +++ .../coretestserver/test_routes/__init__.py | 24 ++ .../coretestserver/test_routes/basic.py | 66 +++++ .../coretestserver/test_routes/encoding.py | 92 ++++++ .../coretestserver/test_routes/errors.py | 28 ++ .../coretestserver/test_routes/helpers.py | 12 + .../coretestserver/test_routes/multipart.py | 88 ++++++ .../coretestserver/test_routes/streams.py | 38 +++ .../coretestserver/test_routes/urlencoded.py | 26 ++ .../coretestserver/test_routes/xml_route.py | 46 +++ .../testserver_tests/coretestserver/setup.py | 35 +++ .../testserver_tests/rest_client_async.py | 0 .../test_rest_context_manager_async.py | 60 ++-- .../test_rest_http_request_async.py | 90 ++++++ .../test_rest_http_response_async.py | 269 ++++++++++++++++++ .../test_rest_stream_responses_async.py | 250 ++++++++++++++++ .../testserver_tests/test_testserver_async.py | 0 .../tests/testserver_tests/conftest.py | 7 - 22 files changed, 1254 insertions(+), 74 deletions(-) create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/conftest.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/__init__.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/setup.py rename sdk/core/azure-core/tests/{ => async_tests}/testserver_tests/rest_client_async.py (100%) rename sdk/core/azure-core/tests/{ => async_tests}/testserver_tests/test_rest_context_manager_async.py (59%) create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_request_async.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_response_async.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_stream_responses_async.py rename sdk/core/azure-core/tests/{ => async_tests}/testserver_tests/test_testserver_async.py (100%) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index 6b556272acc7..d205aaf55d0f 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -180,10 +180,14 @@ async def _make_pipeline_call(self, request, stream, **kwargs): ) response = pipeline_response.http_response if rest_request: - response = response._to_rest_response() - if not kwargs.get("stream", False): - await response.read() - await response.close() + rest_response = response._to_rest_response() + if not stream: + # in this case, the pipeline transport response already called .load_body(), so + # the body is loaded. instead of doing response.read(), going to set the body + # to the internal content + rest_response._content = response.body() # pylint: disable=protected-access + await rest_response.close() + response = rest_response if return_pipeline_response: pipeline_response.http_response = response return pipeline_response diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index a6ddfb6f2a98..c130508aa7bf 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -34,7 +34,7 @@ async def await_result(func, *args, **kwargs): return await result # type: ignore return result -async def _stream_download_helper( +def _stream_download_helper( decompress: bool, stream_download_generator: Callable, response, @@ -46,45 +46,33 @@ async def _stream_download_helper( raise StreamClosedError() response.is_stream_consumed = True - stream_download = stream_download_generator( + return stream_download_generator( pipeline=None, response=response, chunk_size=chunk_size, decompress=decompress, ) - async for part in stream_download: - response._num_bytes_downloaded += len(part) - yield part -async def iter_bytes_helper( +def iter_bytes_helper( stream_download_generator: Callable, response, chunk_size: Optional[int] = None, ) -> AsyncIterator[bytes]: - content = response._get_content() # pylint: disable=protected-access - if content is not None: - if chunk_size is None: - chunk_size = len(content) - for i in range(0, len(content), chunk_size): - yield content[i: i + chunk_size] - else: - async for raw_bytes in _stream_download_helper( - decompress=True, - stream_download_generator=stream_download_generator, - response=response, - chunk_size=chunk_size - ): - yield raw_bytes + return _stream_download_helper( + decompress=True, + stream_download_generator=stream_download_generator, + response=response, + chunk_size=chunk_size + ) -async def iter_raw_helper( +def iter_raw_helper( stream_download_generator: Callable, response, chunk_size: Optional[int] = None ) -> AsyncIterator[bytes]: - async for raw_bytes in _stream_download_helper( + return _stream_download_helper( decompress=False, stream_download_generator=stream_download_generator, response=response, chunk_size=chunk_size - ): - yield raw_bytes + ) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 0c6963177619..2a7ec053e79e 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -417,7 +417,7 @@ def text(self) -> str: return content.decode(encoding) - async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response @@ -431,18 +431,25 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # typ yield part await self.close() - async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_bytes_helper( - stream_download_generator=AioHttpStreamDownloadGenerator, - response=self, - chunk_size=chunk_size, - ): - yield part + content = self._get_content() # pylint: disable=protected-access + if content is not None: + if chunk_size is None: + chunk_size = len(content) + for i in range(0, len(content), chunk_size): + yield content[i: i + chunk_size] + else: + async for raw_bytes in iter_bytes_helper( + stream_download_generator=AioHttpStreamDownloadGenerator, + response=self, + chunk_size=chunk_size + ): + yield raw_bytes await self.close() def __getstate__(self): diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/conftest.py new file mode 100644 index 000000000000..76ec972d42b2 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/conftest.py @@ -0,0 +1,93 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import time +import pytest +import signal +import os +import subprocess +import sys +import random +from six.moves import urllib +from rest_client_async import AsyncTestRestClient + +def is_port_available(port_num): + req = urllib.request.Request("http://localhost:{}/health".format(port_num)) + try: + return urllib.request.urlopen(req).code != 200 + except Exception as e: + return True + +def get_port(): + count = 3 + for _ in range(count): + port_num = random.randrange(3000, 5000) + if is_port_available(port_num): + return port_num + raise TypeError("Tried {} times, can't find an open port".format(count)) + +@pytest.fixture +def port(): + return os.environ["FLASK_PORT"] + +def start_testserver(): + port = get_port() + os.environ["FLASK_APP"] = "coretestserver" + os.environ["FLASK_PORT"] = str(port) + cmd = "flask run -p {}".format(port) + if os.name == 'nt': #On windows, subprocess creation works without being in the shell + child_process = subprocess.Popen(cmd, env=dict(os.environ)) + else: + #On linux, have to set shell=True + child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid, env=dict(os.environ)) + count = 5 + for _ in range(count): + if not is_port_available(port): + return child_process + time.sleep(1) + raise ValueError("Didn't start!") + +def terminate_testserver(process): + if os.name == 'nt': + process.kill() + else: + os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups + +@pytest.fixture(autouse=True, scope="package") +def testserver(): + """Start the Autorest testserver.""" + server = start_testserver() + yield + terminate_testserver(server) + + +# Ignore collection of async tests for Python 2 +collect_ignore_glob = [] +if sys.version_info < (3, 5): + collect_ignore_glob.append("*_async.py") + +@pytest.fixture +def client(port): + return AsyncTestRestClient(port) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/__init__.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/__init__.py new file mode 100644 index 000000000000..63560847a01f --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/__init__.py @@ -0,0 +1,33 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +from flask import Flask, Response +from .test_routes import ( + basic_api, + encoding_api, + errors_api, + streams_api, + urlencoded_api, + multipart_api, + xml_api +) + +app = Flask(__name__) +app.register_blueprint(basic_api, url_prefix="/basic") +app.register_blueprint(encoding_api, url_prefix="/encoding") +app.register_blueprint(errors_api, url_prefix="/errors") +app.register_blueprint(streams_api, url_prefix="/streams") +app.register_blueprint(urlencoded_api, url_prefix="/urlencoded") +app.register_blueprint(multipart_api, url_prefix="/multipart") +app.register_blueprint(xml_api, url_prefix="/xml") + +@app.route('/health', methods=['GET']) +def latin_1_charset_utf8(): + return Response(status=200) + +if __name__ == "__main__": + app.run(debug=True) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py new file mode 100644 index 000000000000..82f4e7ac4566 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py @@ -0,0 +1,24 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +from .basic import basic_api +from .encoding import encoding_api +from .errors import errors_api +from .multipart import multipart_api +from .streams import streams_api +from .urlencoded import urlencoded_api +from .xml_route import xml_api + +__all__ = [ + "basic_api", + "encoding_api", + "errors_api", + "multipart_api", + "streams_api", + "urlencoded_api", + "xml_api", +] diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py new file mode 100644 index 000000000000..4b7d5ae92ad4 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py @@ -0,0 +1,66 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +from flask import ( + Response, + Blueprint, + request +) + +basic_api = Blueprint('basic_api', __name__) + +@basic_api.route('/string', methods=['GET']) +def string(): + return Response( + "Hello, world!", status=200, mimetype="text/plain" + ) + +@basic_api.route('/lines', methods=['GET']) +def lines(): + return Response( + "Hello,\nworld!", status=200, mimetype="text/plain" + ) + +@basic_api.route("/bytes", methods=['GET']) +def bytes(): + return Response( + "Hello, world!".encode(), status=200, mimetype="text/plain" + ) + +@basic_api.route("/html", methods=['GET']) +def html(): + return Response( + "Hello, world!", status=200, mimetype="text/html" + ) + +@basic_api.route("/json", methods=['GET']) +def json(): + return Response( + '{"greeting": "hello", "recipient": "world"}', status=200, mimetype="application/json" + ) + +@basic_api.route("/complicated-json", methods=['POST']) +def complicated_json(): + # thanks to Sean Kane for this test! + assert request.json['EmptyByte'] == '' + assert request.json['EmptyUnicode'] == '' + assert request.json['SpacesOnlyByte'] == ' ' + assert request.json['SpacesOnlyUnicode'] == ' ' + assert request.json['SpacesBeforeByte'] == ' Text' + assert request.json['SpacesBeforeUnicode'] == ' Text' + assert request.json['SpacesAfterByte'] == 'Text ' + assert request.json['SpacesAfterUnicode'] == 'Text ' + assert request.json['SpacesBeforeAndAfterByte'] == ' Text ' + assert request.json['SpacesBeforeAndAfterUnicode'] == ' Text ' + assert request.json['啊齄丂狛'] == 'ꀕ' + assert request.json['RowKey'] == 'test2' + assert request.json['啊齄丂狛狜'] == 'hello' + assert request.json["singlequote"] == "a''''b" + assert request.json["doublequote"] == 'a""""b' + assert request.json["None"] == None + + return Response(status=200) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py new file mode 100644 index 000000000000..12224e568ee5 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py @@ -0,0 +1,92 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from json import dumps +from flask import ( + Response, + Blueprint, +) + +encoding_api = Blueprint('encoding_api', __name__) + +@encoding_api.route('/latin-1', methods=['GET']) +def latin_1(): + r = Response( + "Latin 1: ÿ".encode("latin-1"), status=200 + ) + r.headers["Content-Type"] = "text/plain; charset=latin-1" + return r + +@encoding_api.route('/latin-1-with-utf-8', methods=['GET']) +def latin_1_charset_utf8(): + r = Response( + "Latin 1: ÿ".encode("latin-1"), status=200 + ) + r.headers["Content-Type"] = "text/plain; charset=utf-8" + return r + +@encoding_api.route('/no-charset', methods=['GET']) +def latin_1_no_charset(): + r = Response( + "Hello, world!", status=200 + ) + r.headers["Content-Type"] = "text/plain" + return r + +@encoding_api.route('/iso-8859-1', methods=['GET']) +def iso_8859_1(): + r = Response( + "Accented: Österreich".encode("iso-8859-1"), status=200 + ) + r.headers["Content-Type"] = "text/plain" + return r + +@encoding_api.route('/emoji', methods=['GET']) +def emoji(): + r = Response( + "👩", status=200 + ) + return r + +@encoding_api.route('/emoji-family-skin-tone-modifier', methods=['GET']) +def emoji_family_skin_tone_modifier(): + r = Response( + "👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987", status=200 + ) + return r + +@encoding_api.route('/korean', methods=['GET']) +def korean(): + r = Response( + "아가", status=200 + ) + return r + +@encoding_api.route('/json', methods=['GET']) +def json(): + data = {"greeting": "hello", "recipient": "world"} + content = dumps(data).encode("utf-16") + r = Response( + content, status=200 + ) + r.headers["Content-Type"] = "application/json; charset=utf-16" + return r + +@encoding_api.route('/invalid-codec-name', methods=['GET']) +def invalid_codec_name(): + r = Response( + "おはようございます。".encode("utf-8"), status=200 + ) + r.headers["Content-Type"] = "text/plain; charset=invalid-codec-name" + return r + +@encoding_api.route('/no-charset', methods=['GET']) +def no_charset(): + r = Response( + "Hello, world!", status=200 + ) + r.headers["Content-Type"] = "text/plain" + return r diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py new file mode 100644 index 000000000000..221f598e063a --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py @@ -0,0 +1,28 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from flask import ( + Response, + Blueprint, +) + +errors_api = Blueprint('errors_api', __name__) + +@errors_api.route('/403', methods=['GET']) +def get_403(): + return Response(status=403) + +@errors_api.route('/500', methods=['GET']) +def get_500(): + return Response(status=500) + +@errors_api.route('/stream', methods=['GET']) +def get_stream(): + class StreamingBody: + def __iter__(self): + yield b"Hello, " + yield b"world!" + return Response(StreamingBody(), status=500) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py new file mode 100644 index 000000000000..46680f65d3f9 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py @@ -0,0 +1,12 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +def assert_with_message(param_name, expected_value, actual_value): + assert expected_value == actual_value, "Expected '{}' to be '{}', got '{}'".format( + param_name, expected_value, actual_value + ) + +__all__ = ["assert_with_message"] diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py new file mode 100644 index 000000000000..236496673a2f --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py @@ -0,0 +1,88 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from copy import copy +from flask import ( + Response, + Blueprint, + request, +) +from .helpers import assert_with_message + +multipart_api = Blueprint('multipart_api', __name__) + +multipart_header_start = "multipart/form-data; boundary=" + +# NOTE: the flask behavior is different for aiohttp and requests +# in requests, we see the file content through request.form +# in aiohttp, we see the file through request.files + +@multipart_api.route('/basic', methods=['POST']) +def basic(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + if request.files: + # aiohttp + assert_with_message("content length", 258, request.content_length) + assert_with_message("num files", 1, len(request.files)) + assert_with_message("has file named fileContent", True, bool(request.files.get('fileContent'))) + file_content = request.files['fileContent'] + assert_with_message("file content type", "application/octet-stream", file_content.content_type) + assert_with_message("file content length", 14, file_content.content_length) + assert_with_message("filename", "fileContent", file_content.filename) + assert_with_message("has content disposition header", True, bool(file_content.headers.get("Content-Disposition"))) + assert_with_message( + "content disposition", + 'form-data; name="fileContent"; filename="fileContent"; filename*=utf-8\'\'fileContent', + file_content.headers["Content-Disposition"] + ) + elif request.form: + # requests + assert_with_message("content length", 184, request.content_length) + assert_with_message("fileContent", "", request.form["fileContent"]) + else: + return Response(status=400) # should be either of these + return Response(status=200) + +@multipart_api.route('/data-and-files', methods=['POST']) +def data_and_files(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + assert_with_message("message", "Hello, world!", request.form["message"]) + assert_with_message("message", "", request.form["fileContent"]) + return Response(status=200) + +@multipart_api.route('/data-and-files-tuple', methods=['POST']) +def data_and_files_tuple(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + assert_with_message("message", ["abc"], request.form["message"]) + assert_with_message("message", [""], request.form["fileContent"]) + return Response(status=200) + +@multipart_api.route('/non-seekable-filelike', methods=['POST']) +def non_seekable_filelike(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + if request.files: + # aiohttp + len_files = len(request.files) + assert_with_message("num files", 1, len_files) + # assert_with_message("content length", 258, request.content_length) + assert_with_message("num files", 1, len(request.files)) + assert_with_message("has file named file", True, bool(request.files.get('file'))) + file = request.files['file'] + assert_with_message("file content type", "application/octet-stream", file.content_type) + assert_with_message("file content length", 14, file.content_length) + assert_with_message("filename", "file", file.filename) + assert_with_message("has content disposition header", True, bool(file.headers.get("Content-Disposition"))) + assert_with_message( + "content disposition", + 'form-data; name="fileContent"; filename="fileContent"; filename*=utf-8\'\'fileContent', + file.headers["Content-Disposition"] + ) + elif request.form: + # requests + assert_with_message("num files", 1, len(request.form)) + else: + return Response(status=400) + return Response(status=200) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py new file mode 100644 index 000000000000..1aeb7c05cc21 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py @@ -0,0 +1,38 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from flask import ( + Response, + Blueprint, +) + +streams_api = Blueprint('streams_api', __name__) + +class StreamingBody: + def __iter__(self): + yield b"Hello, " + yield b"world!" + + +def streaming_body(): + yield b"Hello, " + yield b"world!" + +def stream_json_error(): + yield '{"error": {"code": "BadRequest", ' + yield' "message": "You made a bad request"}}' + +@streams_api.route('/basic', methods=['GET']) +def basic(): + return Response(streaming_body(), status=200) + +@streams_api.route('/iterable', methods=['GET']) +def iterable(): + return Response(StreamingBody(), status=200) + +@streams_api.route('/error', methods=['GET']) +def error(): + return Response(stream_json_error(), status=400) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py new file mode 100644 index 000000000000..4ea2bdd2795d --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py @@ -0,0 +1,26 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from flask import ( + Response, + Blueprint, + request, +) +from .helpers import assert_with_message + +urlencoded_api = Blueprint('urlencoded_api', __name__) + +@urlencoded_api.route('/pet/add/', methods=['POST']) +def basic(pet_id): + assert_with_message("pet_id", "1", pet_id) + assert_with_message("content type", "application/x-www-form-urlencoded", request.content_type) + assert_with_message("content length", 47, request.content_length) + assert len(request.form) == 4 + assert_with_message("pet_type", "dog", request.form["pet_type"]) + assert_with_message("pet_food", "meat", request.form["pet_food"]) + assert_with_message("name", "Fido", request.form["name"]) + assert_with_message("pet_age", "42", request.form["pet_age"]) + return Response(status=200) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py new file mode 100644 index 000000000000..c19aed97b6b5 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py @@ -0,0 +1,46 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +import xml.etree.ElementTree as ET +from flask import ( + Response, + Blueprint, + request, +) +from .helpers import assert_with_message + +xml_api = Blueprint('xml_api', __name__) + +@xml_api.route('/basic', methods=['GET', 'PUT']) +def basic(): + basic_body = """ + + + Wake up to WonderWidgets! + + + Overview + Why WonderWidgets are great + + Who buys WonderWidgets + +""" + + if request.method == 'GET': + return Response(basic_body, status=200) + elif request.method == 'PUT': + assert_with_message("content length", str(len(request.data)), request.headers["Content-Length"]) + parsed_xml = ET.fromstring(request.data.decode("utf-8")) + assert_with_message("tag", "slideshow", parsed_xml.tag) + attributes = parsed_xml.attrib + assert_with_message("title attribute", "Sample Slide Show", attributes['title']) + assert_with_message("date attribute", "Date of publication", attributes['date']) + assert_with_message("author attribute", "Yours Truly", attributes['author']) + return Response(status=200) + return Response("You have passed in method '{}' that is not 'GET' or 'PUT'".format(request.method), status=400) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/setup.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/setup.py new file mode 100644 index 000000000000..a43288221498 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/setup.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from setuptools import setup, find_packages + +version = "1.0.0b1" + +setup( + name="coretestserver", + version=version, + include_package_data=True, + description='Testserver for Python Core', + long_description='Testserver for Python Core', + license='MIT License', + author='Microsoft Corporation', + author_email='azpysdkhelp@microsoft.com', + url='https://github.com/iscai-msft/core.testserver', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'License :: OSI Approved :: MIT License', + ], + packages=find_packages(), + install_requires=[ + "flask" + ] +) diff --git a/sdk/core/azure-core/tests/testserver_tests/rest_client_async.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/rest_client_async.py similarity index 100% rename from sdk/core/azure-core/tests/testserver_tests/rest_client_async.py rename to sdk/core/azure-core/tests/async_tests/testserver_tests/rest_client_async.py diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager_async.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_context_manager_async.py similarity index 59% rename from sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager_async.py rename to sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_context_manager_async.py index 9f92fc87a924..4afcac42172f 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager_async.py +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_context_manager_async.py @@ -9,10 +9,6 @@ from azure.core.rest import HttpRequest from rest_client_async import AsyncTestRestClient -@pytest.fixture -def client(port): - return AsyncTestRestClient(port) - @pytest.mark.asyncio async def test_normal_call(client): async def _raise_and_get_text(response): @@ -54,31 +50,33 @@ async def _raise_and_get_text(response): async with response as response: await _raise_and_get_text(response) -@pytest.mark.asyncio -async def test_stream_with_error(client): - request = HttpRequest("GET", url="/streams/error") - async with client.send_request(request, stream=True) as response: - assert not response.is_closed - with pytest.raises(HttpResponseError) as e: - response.raise_for_status() - error = e.value - assert error.status_code == 400 - assert error.reason == "BAD REQUEST" - assert "Operation returned an invalid status 'BAD REQUEST'" in str(error) - with pytest.raises(ResponseNotReadError): - error.error - with pytest.raises(ResponseNotReadError): - error.model - with pytest.raises(ResponseNotReadError): - response.json() - with pytest.raises(ResponseNotReadError): - response.content +# TODO: commenting until https://github.com/Azure/azure-sdk-for-python/issues/18086 is fixed - # NOW WE READ THE RESPONSE - await response.read() - assert error.status_code == 400 - assert error.reason == "BAD REQUEST" - assert error.error.code == "BadRequest" - assert error.error.message == "You made a bad request" - assert error.model.code == "BadRequest" - assert error.error.message == "You made a bad request" \ No newline at end of file +# @pytest.mark.asyncio +# async def test_stream_with_error(client): +# request = HttpRequest("GET", url="/streams/error") +# async with client.send_request(request, stream=True) as response: +# assert not response.is_closed +# with pytest.raises(HttpResponseError) as e: +# response.raise_for_status() +# error = e.value +# assert error.status_code == 400 +# assert error.reason == "BAD REQUEST" +# assert "Operation returned an invalid status 'BAD REQUEST'" in str(error) +# with pytest.raises(ResponseNotReadError): +# error.error +# with pytest.raises(ResponseNotReadError): +# error.model +# with pytest.raises(ResponseNotReadError): +# response.json() +# with pytest.raises(ResponseNotReadError): +# response.content + +# # NOW WE READ THE RESPONSE +# await response.read() +# assert error.status_code == 400 +# assert error.reason == "BAD REQUEST" +# assert error.error.code == "BadRequest" +# assert error.error.message == "You made a bad request" +# assert error.model.code == "BadRequest" +# assert error.error.message == "You made a bad request" \ No newline at end of file diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_request_async.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_request_async.py new file mode 100644 index 000000000000..67f9d419fb31 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_request_async.py @@ -0,0 +1,90 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +# NOTE: These tests are heavily inspired from the httpx test suite: https://github.com/encode/httpx/tree/master/tests +# Thank you httpx for your wonderful tests! +import pytest +from azure.core.rest import HttpRequest +from typing import AsyncGenerator +import collections.abc + +@pytest.fixture +def assert_aiterator_body(): + async def _comparer(request, final_value): + parts = [] + async for part in request.content: + parts.append(part) + content = b"".join(parts) + assert content == final_value + return _comparer + +def test_transfer_encoding_header(): + async def streaming_body(data): + yield data # pragma: nocover + + data = streaming_body(b"test 123") + + request = HttpRequest("POST", "http://example.org", data=data) + assert "Content-Length" not in request.headers + +def test_override_content_length_header(): + async def streaming_body(data): + yield data # pragma: nocover + + data = streaming_body(b"test 123") + headers = {"Content-Length": "0"} + + request = HttpRequest("POST", "http://example.org", data=data, headers=headers) + assert request.headers["Content-Length"] == "0" + +@pytest.mark.asyncio +async def test_aiterbale_content(assert_aiterator_body): + class Content: + async def __aiter__(self): + yield b"test 123" + + request = HttpRequest("POST", "http://example.org", content=Content()) + assert request.headers == {} + await assert_aiterator_body(request, b"test 123") + +@pytest.mark.asyncio +async def test_aiterator_content(assert_aiterator_body): + async def hello_world(): + yield b"Hello, " + yield b"world!" + + request = HttpRequest("POST", url="http://example.org", content=hello_world()) + assert not isinstance(request._data, collections.abc.Iterable) + assert isinstance(request._data, collections.abc.AsyncIterable) + + assert request.headers == {} + await assert_aiterator_body(request, b"Hello, world!") + + # Support 'data' for compat with requests. + request = HttpRequest("POST", url="http://example.org", data=hello_world()) + assert not isinstance(request._data, collections.abc.Iterable) + assert isinstance(request._data, collections.abc.AsyncIterable) + + assert request.headers == {} + await assert_aiterator_body(request, b"Hello, world!") + + # transfer encoding should not be set for GET requests + request = HttpRequest("GET", url="http://example.org", data=hello_world()) + assert not isinstance(request._data, collections.abc.Iterable) + assert isinstance(request._data, collections.abc.AsyncIterable) + + assert request.headers == {} + await assert_aiterator_body(request, b"Hello, world!") + +@pytest.mark.asyncio +async def test_read_content(assert_aiterator_body): + async def content(): + yield b"test 123" + + request = HttpRequest("POST", "http://example.org", content=content()) + await assert_aiterator_body(request, b"test 123") + # in this case, request._data is what we end up passing to the requests transport + assert isinstance(request._data, collections.abc.AsyncIterable) \ No newline at end of file diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_response_async.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_response_async.py new file mode 100644 index 000000000000..cae733c24cac --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_response_async.py @@ -0,0 +1,269 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +# NOTE: These tests are heavily inspired from the httpx test suite: https://github.com/encode/httpx/tree/master/tests +# Thank you httpx for your wonderful tests! +import io +import pytest +from azure.core.rest import HttpRequest +from azure.core.exceptions import HttpResponseError + +@pytest.fixture +def send_request(client): + async def _send_request(request): + response = await client.send_request(request, stream=False) + response.raise_for_status() + return response + return _send_request + +@pytest.mark.asyncio +async def test_response(send_request): + response = await send_request( + HttpRequest("GET", "http://localhost:5000/basic/string"), + ) + assert response.status_code == 200 + assert response.reason == "OK" + assert response.content == b"Hello, world!" + assert response.text == "Hello, world!" + assert response.request.method == "GET" + assert response.request.url == "http://localhost:5000/basic/string" + +@pytest.mark.asyncio +async def test_response_content(send_request): + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/bytes"), + ) + assert response.status_code == 200 + assert response.reason == "OK" + content = await response.read() + assert content == b"Hello, world!" + assert response.text == "Hello, world!" + +@pytest.mark.asyncio +async def test_response_text(send_request): + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/string"), + ) + assert response.status_code == 200 + assert response.reason == "OK" + content = await response.read() + assert content == b"Hello, world!" + assert response.text == "Hello, world!" + assert response.headers["Content-Length"] == '13' + assert response.headers['Content-Type'] == "text/plain; charset=utf-8" + +@pytest.mark.asyncio +async def test_response_html(send_request): + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/html"), + ) + assert response.status_code == 200 + assert response.reason == "OK" + content = await response.read() + assert content == b"Hello, world!" + assert response.text == "Hello, world!" + +@pytest.mark.asyncio +async def test_raise_for_status(client): + # response = await client.send_request( + # HttpRequest("GET", "http://localhost:5000/basic/string"), + # ) + # response.raise_for_status() + + response = await client.send_request( + HttpRequest("GET", "http://localhost:5000/errors/403"), + ) + assert response.status_code == 403 + with pytest.raises(HttpResponseError): + response.raise_for_status() + + response = await client.send_request( + HttpRequest("GET", "http://localhost:5000/errors/500"), + retry_total=0, # takes too long with retires on 500 + ) + assert response.status_code == 500 + with pytest.raises(HttpResponseError): + response.raise_for_status() + +@pytest.mark.asyncio +async def test_response_repr(send_request): + response = await send_request( + HttpRequest("GET", "http://localhost:5000/basic/string") + ) + assert repr(response) == "" + +@pytest.mark.asyncio +async def test_response_content_type_encoding(send_request): + """ + Use the charset encoding in the Content-Type header if possible. + """ + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1") + ) + await response.read() + assert response.content_type == "text/plain; charset=latin-1" + assert response.content == b'Latin 1: \xff' + assert response.text == "Latin 1: ÿ" + assert response.encoding == "latin-1" + + +@pytest.mark.asyncio +async def test_response_autodetect_encoding(send_request): + """ + Autodetect encoding if there is no Content-Type header. + """ + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1") + ) + await response.read() + assert response.text == u'Latin 1: ÿ' + assert response.encoding == "latin-1" + + +@pytest.mark.asyncio +async def test_response_fallback_to_autodetect(send_request): + """ + Fallback to autodetection if we get an invalid charset in the Content-Type header. + """ + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/invalid-codec-name") + ) + await response.read() + assert response.headers["Content-Type"] == "text/plain; charset=invalid-codec-name" + assert response.text == "おはようございます。" + assert response.encoding is None + + +@pytest.mark.asyncio +async def test_response_no_charset_with_ascii_content(send_request): + """ + A response with ascii encoded content should decode correctly, + even with no charset specified. + """ + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/no-charset"), + ) + + assert response.headers["Content-Type"] == "text/plain" + assert response.status_code == 200 + assert response.encoding is None + content = await response.read() + assert content == b"Hello, world!" + assert response.text == "Hello, world!" + + +@pytest.mark.asyncio +async def test_response_no_charset_with_iso_8859_1_content(send_request): + """ + A response with ISO 8859-1 encoded content should decode correctly, + even with no charset specified. + """ + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/iso-8859-1"), + ) + await response.read() + assert response.text == u"Accented: Österreich" + assert response.encoding is None + +# NOTE: aiohttp isn't liking this +# @pytest.mark.asyncio +# async def test_response_set_explicit_encoding(send_request): +# response = await send_request( +# request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1-with-utf-8"), +# ) +# assert response.headers["Content-Type"] == "text/plain; charset=utf-8" +# response.encoding = "latin-1" +# await response.read() +# assert response.text == "Latin 1: ÿ" +# assert response.encoding == "latin-1" + +@pytest.mark.asyncio +async def test_json(send_request): + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/basic/json"), + ) + await response.read() + assert response.json() == {"greeting": "hello", "recipient": "world"} + assert response.encoding is None + +@pytest.mark.asyncio +async def test_json_with_specified_encoding(send_request): + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/json"), + ) + await response.read() + assert response.json() == {"greeting": "hello", "recipient": "world"} + assert response.encoding == "utf-16" + +@pytest.mark.asyncio +async def test_emoji(send_request): + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/emoji"), + ) + await response.read() + assert response.text == "👩" + +@pytest.mark.asyncio +async def test_emoji_family_with_skin_tone_modifier(send_request): + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/emoji-family-skin-tone-modifier"), + ) + await response.read() + assert response.text == "👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987" + +@pytest.mark.asyncio +async def test_korean_nfc(send_request): + response = await send_request( + request=HttpRequest("GET", "http://localhost:5000/encoding/korean"), + ) + await response.read() + assert response.text == "아가" + +@pytest.mark.asyncio +async def test_urlencoded_content(send_request): + await send_request( + request=HttpRequest( + "POST", + "http://localhost:5000/urlencoded/pet/add/1", + data={ "pet_type": "dog", "pet_food": "meat", "name": "Fido", "pet_age": 42 } + ), + ) + +@pytest.mark.asyncio +async def test_multipart_files_content(send_request): + request = HttpRequest( + "POST", + "http://localhost:5000/multipart/basic", + files={"fileContent": io.BytesIO(b"")}, + ) + await send_request(request) + +# @pytest.mark.asyncio +# async def test_multipart_encode_non_seekable_filelike(send_request): +# """ +# Test that special readable but non-seekable filelike objects are supported, +# at the cost of reading them into memory at most once. +# """ + +# class IteratorIO(io.IOBase): +# def __init__(self, iterator): +# self._iterator = iterator + +# def read(self, *args): +# return b"".join(self._iterator) + +# def data(): +# yield b"Hello" +# yield b"World" + +# fileobj = IteratorIO(data()) +# files = {"file": fileobj} +# request = HttpRequest( +# "POST", +# "http://localhost:5000/multipart/non-seekable-filelike", +# files=files, +# ) +# await send_request(request) \ No newline at end of file diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_stream_responses_async.py new file mode 100644 index 000000000000..4c54843d7708 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_stream_responses_async.py @@ -0,0 +1,250 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from azure.core.exceptions import HttpResponseError, ServiceRequestError +import functools +import os +import json +import pytest +from azure.core.rest import HttpRequest +from azure.core.exceptions import StreamClosedError, StreamConsumedError, ResponseNotReadError + +@pytest.mark.asyncio +async def test_iter_raw(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + async with client.send_request(request, stream=True) as response: + raw = b"" + async for part in response.iter_raw(): + raw += part + assert raw == b"Hello, world!" + +@pytest.mark.asyncio +async def test_iter_raw_on_iterable(client): + request = HttpRequest("GET", "http://localhost:5000/streams/iterable") + + async with client.send_request(request, stream=True) as response: + raw = b"" + async for part in response.iter_raw(): + raw += part + assert raw == b"Hello, world!" + +@pytest.mark.asyncio +async def test_iter_with_error(client): + request = HttpRequest("GET", "http://localhost:5000/errors/403") + + async with client.send_request(request, stream=True) as response: + try: + response.raise_for_status() + except HttpResponseError as e: + pass + assert response.is_closed + + try: + async with client.send_request(request, stream=True) as response: + response.raise_for_status() + except HttpResponseError as e: + pass + + assert response.is_closed + + request = HttpRequest("GET", "http://doesNotExist") + with pytest.raises(ServiceRequestError): + async with (await client.send_request(request, stream=True)): + raise ValueError("Should error before entering") + assert response.is_closed + +@pytest.mark.asyncio +async def test_iter_raw_with_chunksize(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + async with client.send_request(request, stream=True) as response: + parts = [] + async for part in response.iter_raw(chunk_size=5): + parts.append(part) + assert parts == [b'Hello', b', wor', b'ld!'] + + async with client.send_request(request, stream=True) as response: + parts = [] + async for part in response.iter_raw(chunk_size=13): + parts.append(part) + assert parts == [b"Hello, world!"] + + async with client.send_request(request, stream=True) as response: + parts = [] + async for part in response.iter_raw(chunk_size=20): + parts.append(part) + assert parts == [b"Hello, world!"] + +@pytest.mark.asyncio +async def test_iter_raw_num_bytes_downloaded(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + async with client.send_request(request, stream=True) as response: + num_downloaded = response.num_bytes_downloaded + async for part in response.iter_raw(): + assert len(part) == (response.num_bytes_downloaded - num_downloaded) + num_downloaded = response.num_bytes_downloaded + +@pytest.mark.asyncio +async def test_iter_bytes(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + async with client.send_request(request, stream=True) as response: + raw = b"" + async for chunk in response.iter_bytes(): + assert response.is_stream_consumed + assert not response.is_closed + raw += chunk + assert response.is_stream_consumed + assert response.is_closed + assert raw == b"Hello, world!" + +@pytest.mark.asyncio +async def test_iter_bytes_with_chunk_size(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + async with client.send_request(request, stream=True) as response: + parts = [] + async for part in response.iter_bytes(chunk_size=5): + parts.append(part) + assert parts == [b"Hello", b", wor", b"ld!"] + + async with client.send_request(request, stream=True) as response: + parts = [] + async for part in response.iter_bytes(chunk_size=13): + parts.append(part) + assert parts == [b"Hello, world!"] + + async with client.send_request(request, stream=True) as response: + parts = [] + async for part in response.iter_bytes(chunk_size=20): + parts.append(part) + assert parts == [b"Hello, world!"] + +@pytest.mark.asyncio +async def test_iter_text(client): + request = HttpRequest("GET", "http://localhost:5000/basic/string") + + async with client.send_request(request, stream=True) as response: + content = "" + async for part in response.iter_text(): + content += part + assert content == "Hello, world!" + +@pytest.mark.asyncio +async def test_iter_text_with_chunk_size(client): + request = HttpRequest("GET", "http://localhost:5000/basic/string") + + async with client.send_request(request, stream=True) as response: + parts = [] + async for part in response.iter_text(chunk_size=5): + parts.append(part) + assert parts == ["Hello", ", wor", "ld!"] + + async with client.send_request(request, stream=True) as response: + parts = [] + async for part in response.iter_text(chunk_size=13): + parts.append(part) + assert parts == ["Hello, world!"] + + async with client.send_request(request, stream=True) as response: + parts = [] + async for part in response.iter_text(chunk_size=20): + parts.append(part) + assert parts == ["Hello, world!"] + +@pytest.mark.asyncio +async def test_iter_lines(client): + request = HttpRequest("GET", "http://localhost:5000/basic/lines") + + async with client.send_request(request, stream=True) as response: + content = [] + async for line in response.iter_lines(): + content.append(line) + assert content == ["Hello,\n", "world!"] + + +@pytest.mark.asyncio +async def test_streaming_response(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + + async with client.send_request(request, stream=True) as response: + assert response.status_code == 200 + assert not response.is_closed + + content = await response.read() + + assert content == b"Hello, world!" + assert response.content == b"Hello, world!" + assert response.is_closed + +@pytest.mark.asyncio +async def test_cannot_read_after_stream_consumed(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + async with client.send_request(request, stream=True) as response: + content = b"" + async for chunk in response.iter_bytes(): + content += chunk + + with pytest.raises(StreamConsumedError) as ex: + await response.read() + assert "You are attempting to read or stream content that has already been streamed" in str(ex.value) + +@pytest.mark.asyncio +async def test_cannot_read_after_response_closed(client): + request = HttpRequest("GET", "http://localhost:5000/streams/basic") + async with client.send_request(request, stream=True) as response: + pass + + with pytest.raises(StreamClosedError) as ex: + await response.read() + assert "You can not try to read or stream this response's content, since the response has been closed" in str(ex.value) + +@pytest.mark.asyncio +async def test_decompress_plain_no_header(client): + # thanks to Xiang Yan for this test! + account_name = "coretests" + url = "https://{}.blob.core.windows.net/tests/test.txt".format(account_name) + request = HttpRequest("GET", url) + async with client: + response = await client.send_request(request, stream=True) + with pytest.raises(ResponseNotReadError): + response.content + await response.read() + assert response.content == b"test" + +@pytest.mark.asyncio +async def test_compress_plain_no_header(client): + # thanks to Xiang Yan for this test! + account_name = "coretests" + url = "https://{}.blob.core.windows.net/tests/test.txt".format(account_name) + request = HttpRequest("GET", url) + async with client: + response = await client.send_request(request, stream=True) + iter = response.iter_raw() + data = b"" + async for d in iter: + data += d + assert data == b"test" + +@pytest.mark.asyncio +async def test_iter_read_back_and_forth(client): + # thanks to McCoy Patiño for this test! + + # while this test may look like it's exposing buggy behavior, this is httpx's behavior + # the reason why the code flow is like this, is because the 'iter_x' functions don't + # actually read the contents into the response, the output them. Once they're yielded, + # the stream is closed, so you have to catch the output when you iterate through it + request = HttpRequest("GET", "http://localhost:5000/basic/lines") + + async with client.send_request(request, stream=True) as response: + async for line in response.iter_lines(): + assert line + with pytest.raises(ResponseNotReadError): + response.text + with pytest.raises(StreamConsumedError): + await response.read() + with pytest.raises(ResponseNotReadError): + response.text \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_testserver_async.py similarity index 100% rename from sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py rename to sdk/core/azure-core/tests/async_tests/testserver_tests/test_testserver_async.py diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 8cb5e80e6d6f..06831c237222 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -28,7 +28,6 @@ import signal import os import subprocess -import sys import random from six.moves import urllib from rest_client import TestRestClient @@ -82,12 +81,6 @@ def testserver(): yield terminate_testserver(server) - -# Ignore collection of async tests for Python 2 -collect_ignore_glob = [] -if sys.version_info < (3, 5): - collect_ignore_glob.append("*_async.py") - @pytest.fixture def client(port): return TestRestClient(port) From f6ba2a9a71c504d78f8980ff49077a7e4583b762 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 12:20:42 -0400 Subject: [PATCH 34/64] fix asynciteratortype typing --- .../pipeline/transport/_requests_asyncio.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index 610d9e4c3596..65fa37d38135 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -200,7 +200,7 @@ class RestAsyncioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsT """Asynchronous streaming of data from the response. """ - async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response @@ -214,16 +214,23 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # typ yield part await self.close() - async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_bytes_helper( - stream_download_generator=AsyncioStreamDownloadGenerator, - response=self, - chunk_size=chunk_size, - ): - yield part + content = self._get_content() # pylint: disable=protected-access + if content is not None: + if chunk_size is None: + chunk_size = len(content) + for i in range(0, len(content), chunk_size): + yield content[i: i + chunk_size] + else: + async for raw_bytes in iter_bytes_helper( + stream_download_generator=AsyncioStreamDownloadGenerator, + response=self, + chunk_size=chunk_size + ): + yield raw_bytes await self.close() From 37ae3927c778f9ce117202cf87d5895aea5a14a3 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 12:24:42 -0400 Subject: [PATCH 35/64] fix aiohttp and trio typing --- .../azure/core/pipeline/transport/_aiohttp.py | 4 ++-- .../core/pipeline/transport/_requests_trio.py | 23 ++++++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 2a7ec053e79e..4d5b84d108a9 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -417,7 +417,7 @@ def text(self) -> str: return content.decode(encoding) - async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: + async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response @@ -431,7 +431,7 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: yield part await self.close() - async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: + async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index 5fdbe4100fbb..93b65c317e87 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -119,7 +119,7 @@ def _to_rest_response(self): class RestTrioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore """Asynchronous streaming of data from the response. """ - async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response @@ -133,18 +133,25 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # typ yield part await self.close() - async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: # type: ignore + async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_bytes_helper( - stream_download_generator=TrioStreamDownloadGenerator, - response=self, - chunk_size=chunk_size, - ): - yield part + content = self._get_content() # pylint: disable=protected-access + if content is not None: + if chunk_size is None: + chunk_size = len(content) + for i in range(0, len(content), chunk_size): + yield content[i: i + chunk_size] + else: + async for raw_bytes in iter_bytes_helper( + stream_download_generator=TrioStreamDownloadGenerator, + response=self, + chunk_size=chunk_size + ): + yield raw_bytes await self.close() async def close(self) -> None: From f23a9b9829c2404ea6964b031bec5f9f1d198197 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 12:36:25 -0400 Subject: [PATCH 36/64] fix kwargs popping of chunk_size --- sdk/core/azure-core/azure/core/pipeline/_tools.py | 2 +- sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py | 4 ++-- .../azure/core/pipeline/transport/_requests_asyncio.py | 4 ++-- .../azure/core/pipeline/transport/_requests_basic.py | 4 ++-- .../azure/core/pipeline/transport/_requests_trio.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools.py b/sdk/core/azure-core/azure/core/pipeline/_tools.py index 125d65b9e314..08870fa226c5 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools.py @@ -87,7 +87,7 @@ def to_rest_response_helper(pipeline_transport_response, response_type): response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access return response -def get_chunk_size(response, **kwargs): +def set_block_size(response, **kwargs): chunk_size = kwargs.pop("chunk_size", None) if not chunk_size: if hasattr(response, "block_size"): diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 4d5b84d108a9..73e43aab239b 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -50,7 +50,7 @@ HttpRequest as RestHttpRequest, AsyncHttpResponse as RestAsyncHttpResponse, ) -from .._tools import to_rest_response_helper, get_chunk_size +from .._tools import to_rest_response_helper, set_block_size from .._tools_async import ( iter_bytes_helper, iter_raw_helper, @@ -224,7 +224,7 @@ def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, *, decompres self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = get_chunk_size(response, **kwargs) + self.block_size = set_block_size(response, chunk_size=kwargs.pop("chunk_size", None), **kwargs) self._decompress = decompress self.content_length = int(response.internal_response.headers.get('Content-Length', 0)) self._decompressor = None diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index 65fa37d38135..5287d066371b 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -44,7 +44,7 @@ _iterate_response_content) from ._requests_basic import RequestsTransportResponse, _read_raw_stream, _RestRequestsTransportResponseBase from ._base_requests_async import RequestsAsyncTransportBase -from .._tools import to_rest_response_helper +from .._tools import to_rest_response_helper, set_block_size from .._tools_async import ( iter_bytes_helper, iter_raw_helper @@ -151,7 +151,7 @@ def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, **kwargs) -> self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = response.block_size + self.block_size = set_block_size(response, chunk_size=kwargs.pop("chunk_size", None), **kwargs) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py index 6a167e3709de..50e5cf1fefe2 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py @@ -55,7 +55,7 @@ to_rest_response_helper, iter_bytes_helper, iter_raw_helper, - get_chunk_size, + set_block_size, ) PipelineType = TypeVar("PipelineType") @@ -140,7 +140,7 @@ def __init__(self, pipeline, response, **kwargs): self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = get_chunk_size(response, **kwargs) + self.block_size = set_block_size(response, chunk_size=kwargs.pop("chunk_size", None), **kwargs) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index 93b65c317e87..fa4f112a5c1f 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -44,7 +44,7 @@ _iterate_response_content) from ._requests_basic import RequestsTransportResponse, _read_raw_stream, _RestRequestsTransportResponseBase from ._base_requests_async import RequestsAsyncTransportBase -from .._tools import to_rest_response_helper +from .._tools import to_rest_response_helper, set_block_size from .._tools_async import ( iter_raw_helper, iter_bytes_helper, @@ -67,7 +67,7 @@ def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, **kwargs) -> self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = response.block_size + self.block_size = set_block_size(response, chunk_size=kwargs.pop("chunk_size", None), **kwargs) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) From 28adf60097b68b1e54378a68eec4e0febd5be282 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 12:55:12 -0400 Subject: [PATCH 37/64] fix deepcopy of requests --- sdk/core/azure-core/azure/core/rest/_rest.py | 7 ++++--- sdk/core/azure-core/azure/core/rest/_rest_py3.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index e4ab3d6a2068..a556a34cc714 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -169,13 +169,14 @@ def __repr__(self): def __deepcopy__(self, memo=None): try: - return HttpRequest( + request = HttpRequest( method=self.method, url=self.url, headers=self.headers, - files=copy.deepcopy(self._files), - data=copy.deepcopy(self._data), ) + request._data = copy.deepcopy(self._data, memo) + request._files = copy.deepcopy(self._files, memo) + return request except (ValueError, TypeError): return copy.copy(self) diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index 9500a24f7f44..bd96a0ae704f 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -195,13 +195,14 @@ def __repr__(self) -> str: def __deepcopy__(self, memo=None) -> "HttpRequest": try: - return HttpRequest( + request = HttpRequest( method=self.method, url=self.url, headers=self.headers, - files=copy.deepcopy(self._files), - data=copy.deepcopy(self._data), ) + request._data = copy.deepcopy(self._data, memo) + request._files = copy.deepcopy(self._files, memo) + return request except (ValueError, TypeError): return copy.copy(self) From 6ffe5db919ee274f5aa1adcab8367c123560fb28 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 13:04:47 -0400 Subject: [PATCH 38/64] say rest requests can only be used with send_Request in changelog, all tests passing --- sdk/core/azure-core/CHANGELOG.md | 4 ++-- .../azure-core/azure/core/pipeline/transport/_aiohttp.py | 6 ++++-- .../azure/core/pipeline/transport/_requests_asyncio.py | 6 ++++-- .../azure/core/pipeline/transport/_requests_trio.py | 6 ++++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index f7f28e4b55af..2be38343a468 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -4,9 +4,9 @@ ### New Features -- Add new provisional module `azure.core.rest`. `azure.core.rest` is our new public simple HTTP library in `azure.core` that users will use to create requests, and consume responses. -- Add new provisional methods `send_request` onto the `azure.core.PipelineClient` and `azure.core.AsyncPipelineClient`. This method takes in +- Add new ***provisional*** methods `send_request` onto the `azure.core.PipelineClient` and `azure.core.AsyncPipelineClient`. This method takes in requests and sends them through our pipelines. +- Add new ***provisional*** module `azure.core.rest`. `azure.core.rest` is our new public simple HTTP library in `azure.core` that users will use to create requests, and consume responses. Can only be used with the provisional method `send_request` on our `PipelineClient`s ## 1.15.0 (2021-06-04) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 73e43aab239b..9e41634602a3 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -428,6 +428,7 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: response=self, chunk_size=chunk_size, ): + self._num_bytes_downloaded += len(part) yield part await self.close() @@ -444,12 +445,13 @@ async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: for i in range(0, len(content), chunk_size): yield content[i: i + chunk_size] else: - async for raw_bytes in iter_bytes_helper( + async for part in iter_bytes_helper( stream_download_generator=AioHttpStreamDownloadGenerator, response=self, chunk_size=chunk_size ): - yield raw_bytes + self._num_bytes_downloaded += len(part) + yield part await self.close() def __getstate__(self): diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index 5287d066371b..68ae1e72f382 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -211,6 +211,7 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: response=self, chunk_size=chunk_size, ): + self._num_bytes_downloaded += len(part) yield part await self.close() @@ -227,10 +228,11 @@ async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: for i in range(0, len(content), chunk_size): yield content[i: i + chunk_size] else: - async for raw_bytes in iter_bytes_helper( + async for part in iter_bytes_helper( stream_download_generator=AsyncioStreamDownloadGenerator, response=self, chunk_size=chunk_size ): - yield raw_bytes + self._num_bytes_downloaded += len(part) + yield part await self.close() diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index fa4f112a5c1f..2b1ee3b60c4e 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -130,6 +130,7 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: response=self, chunk_size=chunk_size, ): + self._num_bytes_downloaded += len(part) yield part await self.close() @@ -146,12 +147,13 @@ async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: for i in range(0, len(content), chunk_size): yield content[i: i + chunk_size] else: - async for raw_bytes in iter_bytes_helper( + async for part in iter_bytes_helper( stream_download_generator=TrioStreamDownloadGenerator, response=self, chunk_size=chunk_size ): - yield raw_bytes + self._num_bytes_downloaded += len(part) + yield part await self.close() async def close(self) -> None: From 10a2350b4283af380a48644b3586b2414a650a8c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 13:28:21 -0400 Subject: [PATCH 39/64] add tests for trio and asyncio transports --- .../tests/async_tests/test_request_asyncio.py | 5 ++- .../test_rest_asyncio_transport.py | 43 +++++++++++++++++++ .../test_rest_trio_transport.py | 41 ++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_asyncio_transport.py create mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_trio_transport.py diff --git a/sdk/core/azure-core/tests/async_tests/test_request_asyncio.py b/sdk/core/azure-core/tests/async_tests/test_request_asyncio.py index 773a320804a4..92097a2265d8 100644 --- a/sdk/core/azure-core/tests/async_tests/test_request_asyncio.py +++ b/sdk/core/azure-core/tests/async_tests/test_request_asyncio.py @@ -26,8 +26,9 @@ async def __anext__(self): raise StopAsyncIteration async with AsyncioRequestsTransport() as transport: - req = HttpRequest('GET', 'http://httpbin.org/post', data=AsyncGen()) - await transport.send(req) + req = HttpRequest('GET', 'http://httpbin.org/anything', data=AsyncGen()) + response = await transport.send(req) + assert json.loads(response.text())['data'] == "azerty" @pytest.mark.asyncio async def test_send_data(): diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_asyncio_transport.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_asyncio_transport.py new file mode 100644 index 000000000000..126487f92aa7 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_asyncio_transport.py @@ -0,0 +1,43 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +import json + +from azure.core.pipeline.transport import AsyncioRequestsTransport +from azure.core.rest import HttpRequest +from rest_client_async import AsyncTestRestClient + +import pytest + + +@pytest.mark.asyncio +async def test_async_gen_data(port): + class AsyncGen: + def __init__(self): + self._range = iter([b"azerty"]) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._range) + except StopIteration: + raise StopAsyncIteration + + async with AsyncioRequestsTransport() as transport: + client = AsyncTestRestClient(port, transport=transport) + request = HttpRequest('GET', 'http://httpbin.org/anything', content=AsyncGen()) + response = await client.send_request(request) + assert response.json()['data'] == "azerty" + +@pytest.mark.asyncio +async def test_send_data(port): + async with AsyncioRequestsTransport() as transport: + client = AsyncTestRestClient(port, transport=transport) + request = HttpRequest('PUT', 'http://httpbin.org/anything', content=b"azerty") + response = await client.send_request(request) + + assert response.json()['data'] == "azerty" \ No newline at end of file diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_trio_transport.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_trio_transport.py new file mode 100644 index 000000000000..7e563ca3d6c5 --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_trio_transport.py @@ -0,0 +1,41 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from azure.core.pipeline.transport import TrioRequestsTransport +from azure.core.rest import HttpRequest +from rest_client_async import AsyncTestRestClient + +import pytest + + +@pytest.mark.trio +async def test_async_gen_data(port): + class AsyncGen: + def __init__(self): + self._range = iter([b"azerty"]) + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self._range) + except StopIteration: + raise StopAsyncIteration + + async with TrioRequestsTransport() as transport: + client = AsyncTestRestClient(port, transport=transport) + request = HttpRequest('GET', 'http://httpbin.org/anything', content=AsyncGen()) + response = await client.send_request(request) + assert response.json()['data'] == "azerty" + +@pytest.mark.trio +async def test_send_data(port): + async with TrioRequestsTransport() as transport: + request = HttpRequest('PUT', 'http://httpbin.org/anything', content=b"azerty") + client = AsyncTestRestClient(port, transport=transport) + response = await client.send_request(request) + + assert response.json()['data'] == "azerty" \ No newline at end of file From 8deb6d7363472f777c2e617b2507653cb8b9d2df Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 14:14:15 -0400 Subject: [PATCH 40/64] lint and mypy --- sdk/core/azure-core/azure/core/_pipeline_client.py | 4 ++-- .../azure-core/azure/core/_pipeline_client_async.py | 8 ++++---- sdk/core/azure-core/azure/core/pipeline/_tools.py | 13 ++++++++++++- .../azure-core/azure/core/pipeline/_tools_async.py | 2 +- sdk/core/azure-core/azure/core/rest/_rest_py3.py | 13 +++++++++++++ 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client.py b/sdk/core/azure-core/azure/core/_pipeline_client.py index 0ddb224b91e2..f39ebbf74b81 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client.py @@ -69,7 +69,7 @@ def _prepare_request(request): # and a bool telling whether we ended up converting it rest_request = False try: - request_to_run = request._to_pipeline_transport_request() + request_to_run = request._to_pipeline_transport_request() # pylint: disable=protected-access rest_request = True except AttributeError: request_to_run = request @@ -198,7 +198,7 @@ def send_request(self, request, **kwargs): pipeline_response = self._pipeline.run(request_to_run, **kwargs) # pylint: disable=protected-access response = pipeline_response.http_response if rest_request: - response = response._to_rest_response() + response = response._to_rest_response() # pylint: disable=protected-access if not kwargs.get("stream", False): response.read() response.close() diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index d205aaf55d0f..1956f04d5bce 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -26,6 +26,7 @@ import logging from collections.abc import Iterable +from typing import Any, Awaitable from .configuration import Configuration from .pipeline import AsyncPipeline from .pipeline.transport._base import PipelineClientBase @@ -37,7 +38,7 @@ AsyncRetryPolicy, ) from ._pipeline_client import _prepare_request -from typing import Any, Awaitable + try: from typing import TYPE_CHECKING except ImportError: @@ -48,7 +49,6 @@ if TYPE_CHECKING: from typing import ( List, - Any, Dict, Union, IO, @@ -180,7 +180,7 @@ async def _make_pipeline_call(self, request, stream, **kwargs): ) response = pipeline_response.http_response if rest_request: - rest_response = response._to_rest_response() + rest_response = response._to_rest_response() # pylint: disable=protected-access if not stream: # in this case, the pipeline transport response already called .load_body(), so # the body is loaded. instead of doing response.read(), going to set the body @@ -208,4 +208,4 @@ def send_request( :rtype: ~azure.core.rest.AsyncHttpResponse """ wrapped = self._make_pipeline_call(request, stream=stream, **kwargs) - return _AsyncContextManager(wrapped=wrapped) \ No newline at end of file + return _AsyncContextManager(wrapped=wrapped) diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools.py b/sdk/core/azure-core/azure/core/pipeline/_tools.py index 08870fa226c5..6653baf72b2a 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools.py @@ -24,6 +24,17 @@ # # -------------------------------------------------------------------------- from ..exceptions import StreamClosedError, StreamConsumedError + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import ( + Callable, + Optional, + Iterator, + ) + from azure.core.rest import HttpResponse + def await_result(func, *args, **kwargs): """If func returns an awaitable, raise that this runner can't handle it.""" result = func(*args, **kwargs) @@ -93,5 +104,5 @@ def set_block_size(response, **kwargs): if hasattr(response, "block_size"): chunk_size = response.block_size elif hasattr(response, "_connection_data_block_size"): - chunk_size = response._connection_data_block_size + chunk_size = response._connection_data_block_size # pylint: disable=protected-access return chunk_size diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index c130508aa7bf..8af2e9266cea 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import Optional, Callable, AsyncIterator, TYPE_CHECKING, Union +from typing import Optional, Callable, AsyncIterator from ..exceptions import StreamClosedError, StreamConsumedError async def await_result(func, *args, **kwargs): diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index bd96a0ae704f..c75e36a05e2d 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -39,6 +39,7 @@ Type, Union, ) +from abc import abstractmethod from azure.core.exceptions import HttpResponseError @@ -440,6 +441,12 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ + # If you don't have a yield in an AsyncIterator function, + # mypy will think it's a coroutine + # see here https://github.com/python/mypy/issues/5385#issuecomment-407281656 + # So, adding this weird yield thing + for _ in []: + yield _ raise NotImplementedError() async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: @@ -449,6 +456,12 @@ async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ + # If you don't have a yield in an AsyncIterator function, + # mypy will think it's a coroutine + # see here https://github.com/python/mypy/issues/5385#issuecomment-407281656 + # So, adding this weird yield thing + for _ in []: + yield _ raise NotImplementedError() async def iter_text(self, chunk_size: int = None) -> AsyncIterator[str]: From 680f423c94e87d35d8638b4db40f47d42f0a87d3 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 14:27:39 -0400 Subject: [PATCH 41/64] move tests, 2.7 passing --- .../coretestserver/coretestserver/__init__.py | 33 ------- .../coretestserver/test_routes/__init__.py | 24 ----- .../coretestserver/test_routes/basic.py | 66 ------------- .../coretestserver/test_routes/encoding.py | 92 ------------------- .../coretestserver/test_routes/errors.py | 28 ------ .../coretestserver/test_routes/helpers.py | 12 --- .../coretestserver/test_routes/multipart.py | 88 ------------------ .../coretestserver/test_routes/streams.py | 38 -------- .../coretestserver/test_routes/urlencoded.py | 26 ------ .../coretestserver/test_routes/xml_route.py | 46 ---------- .../testserver_tests/coretestserver/setup.py | 35 ------- .../async_tests}/conftest.py | 7 ++ .../async_tests}/rest_client_async.py | 0 .../test_rest_asyncio_transport.py | 0 .../test_rest_context_manager_async.py | 0 .../test_rest_http_request_async.py | 0 .../test_rest_http_response_async.py | 0 .../test_rest_stream_responses_async.py | 0 .../async_tests}/test_rest_trio_transport.py | 0 .../async_tests}/test_testserver_async.py | 0 .../tests/testserver_tests/conftest.py | 6 ++ .../test_rest_http_response.py | 2 + 22 files changed, 15 insertions(+), 488 deletions(-) delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/__init__.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py delete mode 100644 sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/setup.py rename sdk/core/azure-core/tests/{async_tests/testserver_tests => testserver_tests/async_tests}/conftest.py (95%) rename sdk/core/azure-core/tests/{async_tests/testserver_tests => testserver_tests/async_tests}/rest_client_async.py (100%) rename sdk/core/azure-core/tests/{async_tests/testserver_tests => testserver_tests/async_tests}/test_rest_asyncio_transport.py (100%) rename sdk/core/azure-core/tests/{async_tests/testserver_tests => testserver_tests/async_tests}/test_rest_context_manager_async.py (100%) rename sdk/core/azure-core/tests/{async_tests/testserver_tests => testserver_tests/async_tests}/test_rest_http_request_async.py (100%) rename sdk/core/azure-core/tests/{async_tests/testserver_tests => testserver_tests/async_tests}/test_rest_http_response_async.py (100%) rename sdk/core/azure-core/tests/{async_tests/testserver_tests => testserver_tests/async_tests}/test_rest_stream_responses_async.py (100%) rename sdk/core/azure-core/tests/{async_tests/testserver_tests => testserver_tests/async_tests}/test_rest_trio_transport.py (100%) rename sdk/core/azure-core/tests/{async_tests/testserver_tests => testserver_tests/async_tests}/test_testserver_async.py (100%) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/__init__.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/__init__.py deleted file mode 100644 index 63560847a01f..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- - -from flask import Flask, Response -from .test_routes import ( - basic_api, - encoding_api, - errors_api, - streams_api, - urlencoded_api, - multipart_api, - xml_api -) - -app = Flask(__name__) -app.register_blueprint(basic_api, url_prefix="/basic") -app.register_blueprint(encoding_api, url_prefix="/encoding") -app.register_blueprint(errors_api, url_prefix="/errors") -app.register_blueprint(streams_api, url_prefix="/streams") -app.register_blueprint(urlencoded_api, url_prefix="/urlencoded") -app.register_blueprint(multipart_api, url_prefix="/multipart") -app.register_blueprint(xml_api, url_prefix="/xml") - -@app.route('/health', methods=['GET']) -def latin_1_charset_utf8(): - return Response(status=200) - -if __name__ == "__main__": - app.run(debug=True) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py deleted file mode 100644 index 82f4e7ac4566..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- - -from .basic import basic_api -from .encoding import encoding_api -from .errors import errors_api -from .multipart import multipart_api -from .streams import streams_api -from .urlencoded import urlencoded_api -from .xml_route import xml_api - -__all__ = [ - "basic_api", - "encoding_api", - "errors_api", - "multipart_api", - "streams_api", - "urlencoded_api", - "xml_api", -] diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py deleted file mode 100644 index 4b7d5ae92ad4..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py +++ /dev/null @@ -1,66 +0,0 @@ -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- - -from flask import ( - Response, - Blueprint, - request -) - -basic_api = Blueprint('basic_api', __name__) - -@basic_api.route('/string', methods=['GET']) -def string(): - return Response( - "Hello, world!", status=200, mimetype="text/plain" - ) - -@basic_api.route('/lines', methods=['GET']) -def lines(): - return Response( - "Hello,\nworld!", status=200, mimetype="text/plain" - ) - -@basic_api.route("/bytes", methods=['GET']) -def bytes(): - return Response( - "Hello, world!".encode(), status=200, mimetype="text/plain" - ) - -@basic_api.route("/html", methods=['GET']) -def html(): - return Response( - "Hello, world!", status=200, mimetype="text/html" - ) - -@basic_api.route("/json", methods=['GET']) -def json(): - return Response( - '{"greeting": "hello", "recipient": "world"}', status=200, mimetype="application/json" - ) - -@basic_api.route("/complicated-json", methods=['POST']) -def complicated_json(): - # thanks to Sean Kane for this test! - assert request.json['EmptyByte'] == '' - assert request.json['EmptyUnicode'] == '' - assert request.json['SpacesOnlyByte'] == ' ' - assert request.json['SpacesOnlyUnicode'] == ' ' - assert request.json['SpacesBeforeByte'] == ' Text' - assert request.json['SpacesBeforeUnicode'] == ' Text' - assert request.json['SpacesAfterByte'] == 'Text ' - assert request.json['SpacesAfterUnicode'] == 'Text ' - assert request.json['SpacesBeforeAndAfterByte'] == ' Text ' - assert request.json['SpacesBeforeAndAfterUnicode'] == ' Text ' - assert request.json['啊齄丂狛'] == 'ꀕ' - assert request.json['RowKey'] == 'test2' - assert request.json['啊齄丂狛狜'] == 'hello' - assert request.json["singlequote"] == "a''''b" - assert request.json["doublequote"] == 'a""""b' - assert request.json["None"] == None - - return Response(status=200) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py deleted file mode 100644 index 12224e568ee5..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py +++ /dev/null @@ -1,92 +0,0 @@ -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- -from json import dumps -from flask import ( - Response, - Blueprint, -) - -encoding_api = Blueprint('encoding_api', __name__) - -@encoding_api.route('/latin-1', methods=['GET']) -def latin_1(): - r = Response( - "Latin 1: ÿ".encode("latin-1"), status=200 - ) - r.headers["Content-Type"] = "text/plain; charset=latin-1" - return r - -@encoding_api.route('/latin-1-with-utf-8', methods=['GET']) -def latin_1_charset_utf8(): - r = Response( - "Latin 1: ÿ".encode("latin-1"), status=200 - ) - r.headers["Content-Type"] = "text/plain; charset=utf-8" - return r - -@encoding_api.route('/no-charset', methods=['GET']) -def latin_1_no_charset(): - r = Response( - "Hello, world!", status=200 - ) - r.headers["Content-Type"] = "text/plain" - return r - -@encoding_api.route('/iso-8859-1', methods=['GET']) -def iso_8859_1(): - r = Response( - "Accented: Österreich".encode("iso-8859-1"), status=200 - ) - r.headers["Content-Type"] = "text/plain" - return r - -@encoding_api.route('/emoji', methods=['GET']) -def emoji(): - r = Response( - "👩", status=200 - ) - return r - -@encoding_api.route('/emoji-family-skin-tone-modifier', methods=['GET']) -def emoji_family_skin_tone_modifier(): - r = Response( - "👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987", status=200 - ) - return r - -@encoding_api.route('/korean', methods=['GET']) -def korean(): - r = Response( - "아가", status=200 - ) - return r - -@encoding_api.route('/json', methods=['GET']) -def json(): - data = {"greeting": "hello", "recipient": "world"} - content = dumps(data).encode("utf-16") - r = Response( - content, status=200 - ) - r.headers["Content-Type"] = "application/json; charset=utf-16" - return r - -@encoding_api.route('/invalid-codec-name', methods=['GET']) -def invalid_codec_name(): - r = Response( - "おはようございます。".encode("utf-8"), status=200 - ) - r.headers["Content-Type"] = "text/plain; charset=invalid-codec-name" - return r - -@encoding_api.route('/no-charset', methods=['GET']) -def no_charset(): - r = Response( - "Hello, world!", status=200 - ) - r.headers["Content-Type"] = "text/plain" - return r diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py deleted file mode 100644 index 221f598e063a..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py +++ /dev/null @@ -1,28 +0,0 @@ -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- -from flask import ( - Response, - Blueprint, -) - -errors_api = Blueprint('errors_api', __name__) - -@errors_api.route('/403', methods=['GET']) -def get_403(): - return Response(status=403) - -@errors_api.route('/500', methods=['GET']) -def get_500(): - return Response(status=500) - -@errors_api.route('/stream', methods=['GET']) -def get_stream(): - class StreamingBody: - def __iter__(self): - yield b"Hello, " - yield b"world!" - return Response(StreamingBody(), status=500) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py deleted file mode 100644 index 46680f65d3f9..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py +++ /dev/null @@ -1,12 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- - -def assert_with_message(param_name, expected_value, actual_value): - assert expected_value == actual_value, "Expected '{}' to be '{}', got '{}'".format( - param_name, expected_value, actual_value - ) - -__all__ = ["assert_with_message"] diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py deleted file mode 100644 index 236496673a2f..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py +++ /dev/null @@ -1,88 +0,0 @@ -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- -from copy import copy -from flask import ( - Response, - Blueprint, - request, -) -from .helpers import assert_with_message - -multipart_api = Blueprint('multipart_api', __name__) - -multipart_header_start = "multipart/form-data; boundary=" - -# NOTE: the flask behavior is different for aiohttp and requests -# in requests, we see the file content through request.form -# in aiohttp, we see the file through request.files - -@multipart_api.route('/basic', methods=['POST']) -def basic(): - assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) - if request.files: - # aiohttp - assert_with_message("content length", 258, request.content_length) - assert_with_message("num files", 1, len(request.files)) - assert_with_message("has file named fileContent", True, bool(request.files.get('fileContent'))) - file_content = request.files['fileContent'] - assert_with_message("file content type", "application/octet-stream", file_content.content_type) - assert_with_message("file content length", 14, file_content.content_length) - assert_with_message("filename", "fileContent", file_content.filename) - assert_with_message("has content disposition header", True, bool(file_content.headers.get("Content-Disposition"))) - assert_with_message( - "content disposition", - 'form-data; name="fileContent"; filename="fileContent"; filename*=utf-8\'\'fileContent', - file_content.headers["Content-Disposition"] - ) - elif request.form: - # requests - assert_with_message("content length", 184, request.content_length) - assert_with_message("fileContent", "", request.form["fileContent"]) - else: - return Response(status=400) # should be either of these - return Response(status=200) - -@multipart_api.route('/data-and-files', methods=['POST']) -def data_and_files(): - assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) - assert_with_message("message", "Hello, world!", request.form["message"]) - assert_with_message("message", "", request.form["fileContent"]) - return Response(status=200) - -@multipart_api.route('/data-and-files-tuple', methods=['POST']) -def data_and_files_tuple(): - assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) - assert_with_message("message", ["abc"], request.form["message"]) - assert_with_message("message", [""], request.form["fileContent"]) - return Response(status=200) - -@multipart_api.route('/non-seekable-filelike', methods=['POST']) -def non_seekable_filelike(): - assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) - if request.files: - # aiohttp - len_files = len(request.files) - assert_with_message("num files", 1, len_files) - # assert_with_message("content length", 258, request.content_length) - assert_with_message("num files", 1, len(request.files)) - assert_with_message("has file named file", True, bool(request.files.get('file'))) - file = request.files['file'] - assert_with_message("file content type", "application/octet-stream", file.content_type) - assert_with_message("file content length", 14, file.content_length) - assert_with_message("filename", "file", file.filename) - assert_with_message("has content disposition header", True, bool(file.headers.get("Content-Disposition"))) - assert_with_message( - "content disposition", - 'form-data; name="fileContent"; filename="fileContent"; filename*=utf-8\'\'fileContent', - file.headers["Content-Disposition"] - ) - elif request.form: - # requests - assert_with_message("num files", 1, len(request.form)) - else: - return Response(status=400) - return Response(status=200) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py deleted file mode 100644 index 1aeb7c05cc21..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py +++ /dev/null @@ -1,38 +0,0 @@ -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- -from flask import ( - Response, - Blueprint, -) - -streams_api = Blueprint('streams_api', __name__) - -class StreamingBody: - def __iter__(self): - yield b"Hello, " - yield b"world!" - - -def streaming_body(): - yield b"Hello, " - yield b"world!" - -def stream_json_error(): - yield '{"error": {"code": "BadRequest", ' - yield' "message": "You made a bad request"}}' - -@streams_api.route('/basic', methods=['GET']) -def basic(): - return Response(streaming_body(), status=200) - -@streams_api.route('/iterable', methods=['GET']) -def iterable(): - return Response(StreamingBody(), status=200) - -@streams_api.route('/error', methods=['GET']) -def error(): - return Response(stream_json_error(), status=400) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py deleted file mode 100644 index 4ea2bdd2795d..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py +++ /dev/null @@ -1,26 +0,0 @@ -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- -from flask import ( - Response, - Blueprint, - request, -) -from .helpers import assert_with_message - -urlencoded_api = Blueprint('urlencoded_api', __name__) - -@urlencoded_api.route('/pet/add/', methods=['POST']) -def basic(pet_id): - assert_with_message("pet_id", "1", pet_id) - assert_with_message("content type", "application/x-www-form-urlencoded", request.content_type) - assert_with_message("content length", 47, request.content_length) - assert len(request.form) == 4 - assert_with_message("pet_type", "dog", request.form["pet_type"]) - assert_with_message("pet_food", "meat", request.form["pet_food"]) - assert_with_message("name", "Fido", request.form["name"]) - assert_with_message("pet_age", "42", request.form["pet_age"]) - return Response(status=200) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py deleted file mode 100644 index c19aed97b6b5..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py +++ /dev/null @@ -1,46 +0,0 @@ -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE.txt in the project root for -# license information. -# ------------------------------------------------------------------------- -import xml.etree.ElementTree as ET -from flask import ( - Response, - Blueprint, - request, -) -from .helpers import assert_with_message - -xml_api = Blueprint('xml_api', __name__) - -@xml_api.route('/basic', methods=['GET', 'PUT']) -def basic(): - basic_body = """ - - - Wake up to WonderWidgets! - - - Overview - Why WonderWidgets are great - - Who buys WonderWidgets - -""" - - if request.method == 'GET': - return Response(basic_body, status=200) - elif request.method == 'PUT': - assert_with_message("content length", str(len(request.data)), request.headers["Content-Length"]) - parsed_xml = ET.fromstring(request.data.decode("utf-8")) - assert_with_message("tag", "slideshow", parsed_xml.tag) - attributes = parsed_xml.attrib - assert_with_message("title attribute", "Sample Slide Show", attributes['title']) - assert_with_message("date attribute", "Date of publication", attributes['date']) - assert_with_message("author attribute", "Yours Truly", attributes['author']) - return Response(status=200) - return Response("You have passed in method '{}' that is not 'GET' or 'PUT'".format(request.method), status=400) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/setup.py b/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/setup.py deleted file mode 100644 index a43288221498..000000000000 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/coretestserver/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python - -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from setuptools import setup, find_packages - -version = "1.0.0b1" - -setup( - name="coretestserver", - version=version, - include_package_data=True, - description='Testserver for Python Core', - long_description='Testserver for Python Core', - license='MIT License', - author='Microsoft Corporation', - author_email='azpysdkhelp@microsoft.com', - url='https://github.com/iscai-msft/core.testserver', - classifiers=[ - 'Development Status :: 4 - Beta', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'License :: OSI Approved :: MIT License', - ], - packages=find_packages(), - install_requires=[ - "flask" - ] -) diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/conftest.py similarity index 95% rename from sdk/core/azure-core/tests/async_tests/testserver_tests/conftest.py rename to sdk/core/azure-core/tests/testserver_tests/async_tests/conftest.py index 76ec972d42b2..17c93009a373 100644 --- a/sdk/core/azure-core/tests/async_tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/conftest.py @@ -91,3 +91,10 @@ def testserver(): @pytest.fixture def client(port): return AsyncTestRestClient(port) + +import sys + +# Ignore collection of async tests for Python 2 +collect_ignore = [] +if sys.version_info < (3, 5): + collect_ignore.append("async_tests") diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/rest_client_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/rest_client_async.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/testserver_tests/rest_client_async.py rename to sdk/core/azure-core/tests/testserver_tests/async_tests/rest_client_async.py diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_asyncio_transport.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_asyncio_transport.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_asyncio_transport.py rename to sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_asyncio_transport.py diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_context_manager_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_context_manager_async.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_context_manager_async.py rename to sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_context_manager_async.py diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_request_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_request_async.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_request_async.py rename to sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_request_async.py diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_response_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_http_response_async.py rename to sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_stream_responses_async.py rename to sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_trio_transport.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_trio_transport.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/testserver_tests/test_rest_trio_transport.py rename to sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_trio_transport.py diff --git a/sdk/core/azure-core/tests/async_tests/testserver_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_testserver_async.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/testserver_tests/test_testserver_async.py rename to sdk/core/azure-core/tests/testserver_tests/async_tests/test_testserver_async.py diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 06831c237222..422904288fd1 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -31,6 +31,12 @@ import random from six.moves import urllib from rest_client import TestRestClient +import sys + +# Ignore collection of async tests for Python 2 +collect_ignore = [] +if sys.version_info < (3, 5): + collect_ignore.append("async_tests") def is_port_available(port_num): req = urllib.request.Request("http://localhost:{}/health".format(port_num)) diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py index 7214f088bca0..d6e12812a79c 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py @@ -220,6 +220,8 @@ def test_multipart_data_and_files_content(send_request): ) send_request(request) +@pytest.mark.skipif(sys.version_info < (3, 0), + reason="In 2.7, get requests error even if we use a pipelien transport") def test_multipart_encode_non_seekable_filelike(send_request): """ Test that special readable but non-seekable filelike objects are supported, From 7793a2a6716781a378209112754d299955080626 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 16:06:32 -0400 Subject: [PATCH 42/64] add provisional docs --- sdk/core/azure-core/CHANGELOG.md | 3 +- .../azure-core/azure/core/_pipeline_client.py | 5 +- .../azure/core/_pipeline_client_async.py | 5 +- sdk/core/azure-core/azure/core/exceptions.py | 20 +++- .../azure-core/azure/core/rest/_helpers.py | 6 +- sdk/core/azure-core/azure/core/rest/_rest.py | 79 +++++++------ .../azure-core/azure/core/rest/_rest_py3.py | 104 +++++++++++------- sdk/core/azure-core/doc/azure.core.rst | 12 ++ 8 files changed, 153 insertions(+), 81 deletions(-) diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 2be38343a468..777301a4eedb 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -7,7 +7,8 @@ - Add new ***provisional*** methods `send_request` onto the `azure.core.PipelineClient` and `azure.core.AsyncPipelineClient`. This method takes in requests and sends them through our pipelines. - Add new ***provisional*** module `azure.core.rest`. `azure.core.rest` is our new public simple HTTP library in `azure.core` that users will use to create requests, and consume responses. Can only be used with the provisional method `send_request` on our `PipelineClient`s - +- Add new ***provisional*** errors `StreamConsumedError`, `StreamClosedError`, and `ResponseNotReadError` to `azure.core.exceptions`. These errors +are thrown if you mishandle streamed responses from the provisional `azure.core.rest` module ## 1.15.0 (2021-06-04) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client.py b/sdk/core/azure-core/azure/core/_pipeline_client.py index f39ebbf74b81..6e6276b42c96 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client.py @@ -186,7 +186,10 @@ def _build_pipeline(self, config, **kwargs): # pylint: disable=no-self-use def send_request(self, request, **kwargs): # type: (HttpRequest, Any) -> HttpResponse - """Runs the network request through the client's chained policies. + """**Provisional** method that runs the network request through the client's chained policies. + + This method is marked as **provisional**, meaning it can be changed + :param request: The network request you want to make. Required. :type request: ~azure.core.rest.HttpRequest :keyword bool stream: Whether the response payload will be streamed. Defaults to False. diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index 1956f04d5bce..5916ba433673 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -200,7 +200,10 @@ def send_request( stream: bool = False, **kwargs: Any ) -> Awaitable[AsyncHttpResponse]: - """Runs the network request through the client's chained policies. + """**Provisional** method that runs the network request through the client's chained policies. + + This method is marked as **provisional**, meaning it can be changed + :param request: The network request you want to make. Required. :type request: ~azure.core.rest.HttpRequest :keyword bool stream: Whether the response payload will be streamed. Defaults to False. diff --git a/sdk/core/azure-core/azure/core/exceptions.py b/sdk/core/azure-core/azure/core/exceptions.py index ddeec110d59f..a2282657b17a 100644 --- a/sdk/core/azure-core/azure/core/exceptions.py +++ b/sdk/core/azure-core/azure/core/exceptions.py @@ -435,6 +435,12 @@ def __str__(self): return super(ODataV4Error, self).__str__() class StreamConsumedError(Exception): + """:bolditalic:`Provisional` error thrown if you try to access the stream of a response once consumed. + + :bolditalic:`This error is provisional`, meaning it may be changed. It is thrown if you try to + read / stream an ~azure.core.rest.HttpResponse or ~azure.core.rest.AsyncHttpResponse once + the response's stream has been consumed + """ def __init__(self): message = ( "You are attempting to read or stream content that has already been streamed. " @@ -443,14 +449,26 @@ def __init__(self): super(StreamConsumedError, self).__init__(message) class StreamClosedError(Exception): + """:bolditalic:`Provisional` error thrown if you try to access the stream of a response once closed. + + :bolditalic:`This error is provisional`, meaning it may be changed. It is thrown if you try to + read / stream an ~azure.core.rest.HttpResponse or ~azure.core.rest.AsyncHttpResponse once + the response's stream has been closed + """ def __init__(self): message = ( "You can not try to read or stream this response's content, since the " - "response has been closed." + "response's stream has been closed." ) super(StreamClosedError, self).__init__(message) class ResponseNotReadError(Exception): + """:bolditalic:`Provisional` error thrown if you try to access a response's content without reading first. + + :bolditalic:`This error is provisional`, meaning it may be changed. It is thrown if you try to + access an ~azure.core.rest.HttpResponse or ~azure.core.rest.AsyncHttpResponse's content without + first reading the response's bytes in first. + """ def __init__(self): message = ( diff --git a/sdk/core/azure-core/azure/core/rest/_helpers.py b/sdk/core/azure-core/azure/core/rest/_helpers.py index 2fcd8e83e7a2..853acb4724e4 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers.py @@ -54,9 +54,9 @@ PrimitiveData = Optional[Union[str, int, float, bool]] -ParamsType = Dict[str, Union[PrimitiveData, Sequence[PrimitiveData]]] +ParamsType = Mapping[str, Union[PrimitiveData, Sequence[PrimitiveData]]] -HeadersType = Dict[str, str] +HeadersType = Mapping[str, str] FileContent = Union[str, bytes, IO[str], IO[bytes]] FileType = Union[ @@ -64,7 +64,7 @@ ] FilesType = Union[ - Dict[str, FileType], + Mapping[str, FileType], Sequence[Tuple[str, FileType]] ] diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index a556a34cc714..2b5131179e83 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -65,33 +65,30 @@ ################################## CLASSES ###################################### class HttpRequest(object): - """Represents an HTTP request. + """Provisional object that represents an HTTP request. - :param method: HTTP method (GET, HEAD, etc.) - :type method: str or ~azure.core.protocol.HttpVerbs + **This object is provisional**, meaning it may be changed. + + :param str method: HTTP method (GET, HEAD, etc.) :param str url: The url for your request - :keyword params: Query parameters to be mapped into your URL. Your input - should be a mapping or sequence of query name to query value(s). - :paramtype params: mapping or sequence - :keyword headers: HTTP headers you want in your request. Your input should - be a mapping or sequence of header name to header value. - :paramtype headers: mapping or sequence - :keyword dict data: Form data you want in your request body. Use for form-encoded data, i.e. - HTML forms. + :keyword mapping params: Query parameters to be mapped into your URL. Your input + should be a mapping of query name to query value(s). + :keyword mapping headers: HTTP headers you want in your request. Your input should + be a mapping of header name to header value. :keyword any json: A JSON serializable object. We handle JSON-serialization for your object, so use this for more complicated data structures than `data`. - :keyword files: Files you want to in your request body. Use for uploading files with - multipart encoding. Your input should be a mapping or sequence of file name to file content. - Use the `data` kwarg in addition if you want to include non-file data files as part of your request. - :paramtype files: mapping or sequence :keyword content: Content you want in your request body. Think of it as the kwarg you should input if your data doesn't fit into `json`, `data`, or `files`. Accepts a bytes type, or a generator that yields bytes. :paramtype content: str or bytes or iterable[bytes] or asynciterable[bytes] + :keyword dict data: Form data you want in your request body. Use for form-encoded data, i.e. + HTML forms. + :keyword mapping files: Files you want to in your request body. Use for uploading files with + multipart encoding. Your input should be a mapping of file name to file content. + Use the `data` kwarg in addition if you want to include non-file data files as part of your request. :ivar str url: The URL this request is against. :ivar str method: The method type of this request. - :ivar headers: The HTTP headers you passed in to your request - :vartype headers: mapping or sequence + :ivar mapping headers: The HTTP headers you passed in to your request :ivar bytes content: The content passed in for the request """ @@ -157,7 +154,10 @@ def _update_headers(self, default_headers): @property def content(self): # type: (...) -> Any - """Gets the request content. + """Get's the request's content + + :return: The request's content + :rtype: any """ return self._data or self._files @@ -188,24 +188,6 @@ def _from_pipeline_transport_request(cls, pipeline_transport_request): return from_pipeline_transport_request_helper(cls, pipeline_transport_request) class _HttpResponseBase(object): # pylint: disable=too-many-instance-attributes - """Class for HttpResponse. - - :keyword request: The request that resulted in this response. - :paramtype request: ~azure.core.rest.HttpRequest - :ivar int status_code: The status code of this response - :ivar headers: The response headers - :vartype headers: dict[str, any] - :ivar str reason: The reason phrase for this response - :ivar bytes content: The response content in bytes - :ivar str url: The URL that resulted in this response - :ivar str encoding: The response encoding. Is settable, by default - is the response Content-Type header - :ivar str text: The response body as a string. - :ivar request: The request that resulted in this response. - :vartype request: ~azure.core.rest.HttpRequest - :ivar str content_type: The content type of the response - :ivar bool is_error: Whether this response is an error. - """ def __init__(self, **kwargs): # type: (Any) -> None @@ -325,6 +307,31 @@ def __repr__(self): ) class HttpResponse(_HttpResponseBase): # pylint: disable=too-many-instance-attributes + """**Provisional** object that represents an HTTP response. + + **This object is provisional**, meaning it may be changed. + + :keyword request: The request that resulted in this response. + :paramtype request: ~azure.core.rest.HttpRequest + :keyword internal_response: The object returned from the HTTP library. + :ivar int status_code: The status code of this response + :ivar mapping headers: The response headers + :ivar str reason: The reason phrase for this response + :ivar bytes content: The response content in bytes. + :ivar str url: The URL that resulted in this response + :ivar str encoding: The response encoding. Is settable, by default + is the response Content-Type header + :ivar str text: The response body as a string. + :ivar request: The request that resulted in this response. + :vartype request: ~azure.core.rest.HttpRequest + :ivar internal_response: The object returned from the HTTP library. + :ivar str content_type: The content type of the response + :ivar bool is_closed: Whether the network connection has been closed yet + :ivar bool is_stream_consumed: When getting a stream response, checks + whether the stream has been fully consumed + :ivar int num_bytes_downloaded: The number of bytes in your stream that + have been downloaded + """ def __enter__(self): # type: (...) -> HttpResponse diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index c75e36a05e2d..61f573394deb 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -87,17 +87,16 @@ async def close(self): ################################## CLASSES ###################################### class HttpRequest: - """Represents an HTTP request. + """**Provisional** object that represents an HTTP request. - :param method: HTTP method (GET, HEAD, etc.) - :type method: str or ~azure.core.protocol.HttpVerbs + **This object is provisional**, meaning it may be changed. + + :param str method: HTTP method (GET, HEAD, etc.) :param str url: The url for your request - :keyword params: Query parameters to be mapped into your URL. Your input - should be a mapping or sequence of query name to query value(s). - :paramtype params: mapping or sequence - :keyword headers: HTTP headers you want in your request. Your input should - be a mapping or sequence of header name to header value. - :paramtype headers: mapping or sequence + :keyword mapping params: Query parameters to be mapped into your URL. Your input + should be a mapping of query name to query value(s). + :keyword mapping headers: HTTP headers you want in your request. Your input should + be a mapping of header name to header value. :keyword any json: A JSON serializable object. We handle JSON-serialization for your object, so use this for more complicated data structures than `data`. :keyword content: Content you want in your request body. Think of it as the kwarg you should input @@ -106,15 +105,13 @@ class HttpRequest: :paramtype content: str or bytes or iterable[bytes] or asynciterable[bytes] :keyword dict data: Form data you want in your request body. Use for form-encoded data, i.e. HTML forms. - :keyword files: Files you want to in your request body. Use for uploading files with - multipart encoding. Your input should be a mapping or sequence of file name to file content. + :keyword mapping files: Files you want to in your request body. Use for uploading files with + multipart encoding. Your input should be a mapping of file name to file content. Use the `data` kwarg in addition if you want to include non-file data files as part of your request. - :paramtype files: mapping or sequence :ivar str url: The URL this request is against. :ivar str method: The method type of this request. - :ivar headers: The HTTP headers you passed in to your request - :vartype headers: mapping or sequence - :ivar bytes content: The content passed in for the request + :ivar mapping headers: The HTTP headers you passed in to your request + :ivar any content: The content passed in for the request """ def __init__( @@ -185,7 +182,10 @@ def _set_body( @property def content(self) -> Any: - """Gets the request content. + """Get's the request's content + + :return: The request's content + :rtype: any """ return self._data or self._files @@ -215,28 +215,6 @@ def _from_pipeline_transport_request(cls, pipeline_transport_request): return from_pipeline_transport_request_helper(cls, pipeline_transport_request) class _HttpResponseBase: # pylint: disable=too-many-instance-attributes - """Base class for HttpResponse and AsyncHttpResponse. - - :keyword request: The request that resulted in this response. - :paramtype request: ~azure.core.rest.HttpRequest - :ivar int status_code: The status code of this response - :ivar headers: The response headers - :vartype headers: dict[str, any] - :ivar str reason: The reason phrase for this response - :ivar bytes content: The response content in bytes - :ivar str url: The URL that resulted in this response - :ivar str encoding: The response encoding. Is settable, by default - is the response Content-Type header - :ivar str text: The response body as a string. - :ivar request: The request that resulted in this response. - :vartype request: ~azure.core.rest.HttpRequest - :ivar str content_type: The content type of the response - :ivar bool is_closed: Whether the network connection has been closed yet - :ivar bool is_stream_consumed: When getting a stream response, checks - whether the stream has been fully consumed - :ivar int num_bytes_downloaded: The number of bytes in your stream that - have been downloaded - """ def __init__( self, @@ -343,6 +321,31 @@ def content(self) -> bytes: return cast(bytes, self._get_content()) class HttpResponse(_HttpResponseBase): + """**Provisional** object that represents an HTTP response. + + **This object is provisional**, meaning it may be changed. + + :keyword request: The request that resulted in this response. + :paramtype request: ~azure.core.rest.HttpRequest + :keyword internal_response: The object returned from the HTTP library. + :ivar int status_code: The status code of this response + :ivar mapping headers: The response headers + :ivar str reason: The reason phrase for this response + :ivar bytes content: The response content in bytes. + :ivar str url: The URL that resulted in this response + :ivar str encoding: The response encoding. Is settable, by default + is the response Content-Type header + :ivar str text: The response body as a string. + :ivar request: The request that resulted in this response. + :vartype request: ~azure.core.rest.HttpRequest + :ivar internal_response: The object returned from the HTTP library. + :ivar str content_type: The content type of the response + :ivar bool is_closed: Whether the network connection has been closed yet + :ivar bool is_stream_consumed: When getting a stream response, checks + whether the stream has been fully consumed + :ivar int num_bytes_downloaded: The number of bytes in your stream that + have been downloaded + """ def __enter__(self) -> "HttpResponse": return self @@ -420,6 +423,31 @@ def __repr__(self) -> str: ) class AsyncHttpResponse(_HttpResponseBase): + """**Provisional** object that represents an Async HTTP response. + + **This object is provisional**, meaning it may be changed. + + :keyword request: The request that resulted in this response. + :paramtype request: ~azure.core.rest.HttpRequest + :keyword internal_response: The object returned from the HTTP library. + :ivar int status_code: The status code of this response + :ivar mapping headers: The response headers + :ivar str reason: The reason phrase for this response + :ivar bytes content: The response content in bytes. + :ivar str url: The URL that resulted in this response + :ivar str encoding: The response encoding. Is settable, by default + is the response Content-Type header + :ivar str text: The response body as a string. + :ivar request: The request that resulted in this response. + :vartype request: ~azure.core.rest.HttpRequest + :ivar internal_response: The object returned from the HTTP library. + :ivar str content_type: The content type of the response + :ivar bool is_closed: Whether the network connection has been closed yet + :ivar bool is_stream_consumed: When getting a stream response, checks + whether the stream has been fully consumed + :ivar int num_bytes_downloaded: The number of bytes in your stream that + have been downloaded + """ async def read(self) -> bytes: """Read the response's bytes into memory. diff --git a/sdk/core/azure-core/doc/azure.core.rst b/sdk/core/azure-core/doc/azure.core.rst index 36716ad49c62..c6f0ed92463b 100644 --- a/sdk/core/azure-core/doc/azure.core.rst +++ b/sdk/core/azure-core/doc/azure.core.rst @@ -72,3 +72,15 @@ azure.core.serialization :members: :undoc-members: :inherited-members: + +azure.core.rest +------------------- +***THIS MODULE IS PROVISIONAL*** + +This module is ***provisional***, meaning any of the objects and methods in this module may be changed. + +.. automodule:: azure.core.rest + :members: + :undoc-members: + :inherited-members: + From 051743c7acdc5d346423ad3f654ad05f0702e68e Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 16:21:59 -0400 Subject: [PATCH 43/64] remove port from rest tests --- .../test_rest_http_response_async.py | 48 ++++++++-------- .../test_rest_stream_responses_async.py | 30 +++++----- .../tests/testserver_tests/conftest.py | 8 --- .../test_rest_http_request.py | 2 +- .../test_rest_http_response.py | 56 +++++++++---------- .../test_rest_stream_responses.py | 32 +++++------ .../testserver_tests/test_testserver_async.py | 37 ------------ 7 files changed, 84 insertions(+), 129 deletions(-) delete mode 100644 sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py index cae733c24cac..e763c886bf81 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py @@ -20,21 +20,21 @@ async def _send_request(request): return _send_request @pytest.mark.asyncio -async def test_response(send_request): +async def test_response(send_request, port): response = await send_request( - HttpRequest("GET", "http://localhost:5000/basic/string"), + HttpRequest("GET", "/basic/string"), ) assert response.status_code == 200 assert response.reason == "OK" assert response.content == b"Hello, world!" assert response.text == "Hello, world!" assert response.request.method == "GET" - assert response.request.url == "http://localhost:5000/basic/string" + assert response.request.url == "http://localhost:{}/basic/string".format(port) @pytest.mark.asyncio async def test_response_content(send_request): response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/bytes"), + request=HttpRequest("GET", "/basic/bytes"), ) assert response.status_code == 200 assert response.reason == "OK" @@ -45,7 +45,7 @@ async def test_response_content(send_request): @pytest.mark.asyncio async def test_response_text(send_request): response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/string"), + request=HttpRequest("GET", "/basic/string"), ) assert response.status_code == 200 assert response.reason == "OK" @@ -58,7 +58,7 @@ async def test_response_text(send_request): @pytest.mark.asyncio async def test_response_html(send_request): response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/html"), + request=HttpRequest("GET", "/basic/html"), ) assert response.status_code == 200 assert response.reason == "OK" @@ -69,19 +69,19 @@ async def test_response_html(send_request): @pytest.mark.asyncio async def test_raise_for_status(client): # response = await client.send_request( - # HttpRequest("GET", "http://localhost:5000/basic/string"), + # HttpRequest("GET", "/basic/string"), # ) # response.raise_for_status() response = await client.send_request( - HttpRequest("GET", "http://localhost:5000/errors/403"), + HttpRequest("GET", "/errors/403"), ) assert response.status_code == 403 with pytest.raises(HttpResponseError): response.raise_for_status() response = await client.send_request( - HttpRequest("GET", "http://localhost:5000/errors/500"), + HttpRequest("GET", "/errors/500"), retry_total=0, # takes too long with retires on 500 ) assert response.status_code == 500 @@ -91,7 +91,7 @@ async def test_raise_for_status(client): @pytest.mark.asyncio async def test_response_repr(send_request): response = await send_request( - HttpRequest("GET", "http://localhost:5000/basic/string") + HttpRequest("GET", "/basic/string") ) assert repr(response) == "" @@ -101,7 +101,7 @@ async def test_response_content_type_encoding(send_request): Use the charset encoding in the Content-Type header if possible. """ response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1") + request=HttpRequest("GET", "/encoding/latin-1") ) await response.read() assert response.content_type == "text/plain; charset=latin-1" @@ -116,7 +116,7 @@ async def test_response_autodetect_encoding(send_request): Autodetect encoding if there is no Content-Type header. """ response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1") + request=HttpRequest("GET", "/encoding/latin-1") ) await response.read() assert response.text == u'Latin 1: ÿ' @@ -129,7 +129,7 @@ async def test_response_fallback_to_autodetect(send_request): Fallback to autodetection if we get an invalid charset in the Content-Type header. """ response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/invalid-codec-name") + request=HttpRequest("GET", "/encoding/invalid-codec-name") ) await response.read() assert response.headers["Content-Type"] == "text/plain; charset=invalid-codec-name" @@ -144,7 +144,7 @@ async def test_response_no_charset_with_ascii_content(send_request): even with no charset specified. """ response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/no-charset"), + request=HttpRequest("GET", "/encoding/no-charset"), ) assert response.headers["Content-Type"] == "text/plain" @@ -162,7 +162,7 @@ async def test_response_no_charset_with_iso_8859_1_content(send_request): even with no charset specified. """ response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/iso-8859-1"), + request=HttpRequest("GET", "/encoding/iso-8859-1"), ) await response.read() assert response.text == u"Accented: Österreich" @@ -172,7 +172,7 @@ async def test_response_no_charset_with_iso_8859_1_content(send_request): # @pytest.mark.asyncio # async def test_response_set_explicit_encoding(send_request): # response = await send_request( -# request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1-with-utf-8"), +# request=HttpRequest("GET", "/encoding/latin-1-with-utf-8"), # ) # assert response.headers["Content-Type"] == "text/plain; charset=utf-8" # response.encoding = "latin-1" @@ -183,7 +183,7 @@ async def test_response_no_charset_with_iso_8859_1_content(send_request): @pytest.mark.asyncio async def test_json(send_request): response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/json"), + request=HttpRequest("GET", "/basic/json"), ) await response.read() assert response.json() == {"greeting": "hello", "recipient": "world"} @@ -192,7 +192,7 @@ async def test_json(send_request): @pytest.mark.asyncio async def test_json_with_specified_encoding(send_request): response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/json"), + request=HttpRequest("GET", "/encoding/json"), ) await response.read() assert response.json() == {"greeting": "hello", "recipient": "world"} @@ -201,7 +201,7 @@ async def test_json_with_specified_encoding(send_request): @pytest.mark.asyncio async def test_emoji(send_request): response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/emoji"), + request=HttpRequest("GET", "/encoding/emoji"), ) await response.read() assert response.text == "👩" @@ -209,7 +209,7 @@ async def test_emoji(send_request): @pytest.mark.asyncio async def test_emoji_family_with_skin_tone_modifier(send_request): response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/emoji-family-skin-tone-modifier"), + request=HttpRequest("GET", "/encoding/emoji-family-skin-tone-modifier"), ) await response.read() assert response.text == "👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987" @@ -217,7 +217,7 @@ async def test_emoji_family_with_skin_tone_modifier(send_request): @pytest.mark.asyncio async def test_korean_nfc(send_request): response = await send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/korean"), + request=HttpRequest("GET", "/encoding/korean"), ) await response.read() assert response.text == "아가" @@ -227,7 +227,7 @@ async def test_urlencoded_content(send_request): await send_request( request=HttpRequest( "POST", - "http://localhost:5000/urlencoded/pet/add/1", + "/urlencoded/pet/add/1", data={ "pet_type": "dog", "pet_food": "meat", "name": "Fido", "pet_age": 42 } ), ) @@ -236,7 +236,7 @@ async def test_urlencoded_content(send_request): async def test_multipart_files_content(send_request): request = HttpRequest( "POST", - "http://localhost:5000/multipart/basic", + "/multipart/basic", files={"fileContent": io.BytesIO(b"")}, ) await send_request(request) @@ -263,7 +263,7 @@ async def test_multipart_files_content(send_request): # files = {"file": fileobj} # request = HttpRequest( # "POST", -# "http://localhost:5000/multipart/non-seekable-filelike", +# "/multipart/non-seekable-filelike", # files=files, # ) # await send_request(request) \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py index 4c54843d7708..15587805c211 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py @@ -13,7 +13,7 @@ @pytest.mark.asyncio async def test_iter_raw(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: raw = b"" async for part in response.iter_raw(): @@ -22,7 +22,7 @@ async def test_iter_raw(client): @pytest.mark.asyncio async def test_iter_raw_on_iterable(client): - request = HttpRequest("GET", "http://localhost:5000/streams/iterable") + request = HttpRequest("GET", "/streams/iterable") async with client.send_request(request, stream=True) as response: raw = b"" @@ -32,7 +32,7 @@ async def test_iter_raw_on_iterable(client): @pytest.mark.asyncio async def test_iter_with_error(client): - request = HttpRequest("GET", "http://localhost:5000/errors/403") + request = HttpRequest("GET", "/errors/403") async with client.send_request(request, stream=True) as response: try: @@ -57,7 +57,7 @@ async def test_iter_with_error(client): @pytest.mark.asyncio async def test_iter_raw_with_chunksize(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: parts = [] @@ -79,7 +79,7 @@ async def test_iter_raw_with_chunksize(client): @pytest.mark.asyncio async def test_iter_raw_num_bytes_downloaded(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: num_downloaded = response.num_bytes_downloaded @@ -89,7 +89,7 @@ async def test_iter_raw_num_bytes_downloaded(client): @pytest.mark.asyncio async def test_iter_bytes(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: raw = b"" @@ -103,7 +103,7 @@ async def test_iter_bytes(client): @pytest.mark.asyncio async def test_iter_bytes_with_chunk_size(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: parts = [] @@ -125,7 +125,7 @@ async def test_iter_bytes_with_chunk_size(client): @pytest.mark.asyncio async def test_iter_text(client): - request = HttpRequest("GET", "http://localhost:5000/basic/string") + request = HttpRequest("GET", "/basic/string") async with client.send_request(request, stream=True) as response: content = "" @@ -135,7 +135,7 @@ async def test_iter_text(client): @pytest.mark.asyncio async def test_iter_text_with_chunk_size(client): - request = HttpRequest("GET", "http://localhost:5000/basic/string") + request = HttpRequest("GET", "/basic/string") async with client.send_request(request, stream=True) as response: parts = [] @@ -157,7 +157,7 @@ async def test_iter_text_with_chunk_size(client): @pytest.mark.asyncio async def test_iter_lines(client): - request = HttpRequest("GET", "http://localhost:5000/basic/lines") + request = HttpRequest("GET", "/basic/lines") async with client.send_request(request, stream=True) as response: content = [] @@ -168,7 +168,7 @@ async def test_iter_lines(client): @pytest.mark.asyncio async def test_streaming_response(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: assert response.status_code == 200 @@ -182,7 +182,7 @@ async def test_streaming_response(client): @pytest.mark.asyncio async def test_cannot_read_after_stream_consumed(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: content = b"" async for chunk in response.iter_bytes(): @@ -194,13 +194,13 @@ async def test_cannot_read_after_stream_consumed(client): @pytest.mark.asyncio async def test_cannot_read_after_response_closed(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: pass with pytest.raises(StreamClosedError) as ex: await response.read() - assert "You can not try to read or stream this response's content, since the response has been closed" in str(ex.value) + assert "You can not try to read or stream this response's content, since the response's stream has been closed" in str(ex.value) @pytest.mark.asyncio async def test_decompress_plain_no_header(client): @@ -237,7 +237,7 @@ async def test_iter_read_back_and_forth(client): # the reason why the code flow is like this, is because the 'iter_x' functions don't # actually read the contents into the response, the output them. Once they're yielded, # the stream is closed, so you have to catch the output when you iterate through it - request = HttpRequest("GET", "http://localhost:5000/basic/lines") + request = HttpRequest("GET", "/basic/lines") async with client.send_request(request, stream=True) as response: async for line in response.iter_lines(): diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index fc7558c5ab39..422904288fd1 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -87,14 +87,6 @@ def testserver(): yield terminate_testserver(server) -<<<<<<< HEAD @pytest.fixture def client(port): return TestRestClient(port) -======= - -# Ignore collection of async tests for Python 2 -collect_ignore_glob = [] -if sys.version_info < (3, 5): - collect_ignore_glob.append("*_async.py") ->>>>>>> 97e5b6e3f554ac224492ecb6bd6d26bcb9a2248c diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py index cde6e83de490..74d815648589 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py @@ -274,7 +274,7 @@ def test_complicated_json(client): "doublequote": 'a""""b', "None": None, } - request = HttpRequest("POST", "http://localhost:5000/basic/complicated-json", json=input) + request = HttpRequest("POST", "/basic/complicated-json", json=input) r = client.send_request(request) r.raise_for_status() diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py index d6e12812a79c..5915e5548c63 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py @@ -22,20 +22,20 @@ def _send_request(request): return response return _send_request -def test_response(send_request): +def test_response(send_request, port): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/string"), + request=HttpRequest("GET", "/basic/string"), ) assert response.status_code == 200 assert response.reason == "OK" assert response.text == "Hello, world!" assert response.request.method == "GET" - assert response.request.url == "http://localhost:5000/basic/string" + assert response.request.url == "http://localhost:{}/basic/string".format(port) def test_response_content(send_request): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/bytes"), + request=HttpRequest("GET", "/basic/bytes"), ) assert response.status_code == 200 assert response.reason == "OK" @@ -44,7 +44,7 @@ def test_response_content(send_request): def test_response_text(send_request): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/string"), + request=HttpRequest("GET", "/basic/string"), ) assert response.status_code == 200 assert response.reason == "OK" @@ -55,7 +55,7 @@ def test_response_text(send_request): def test_response_html(send_request): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/html"), + request=HttpRequest("GET", "/basic/html"), ) assert response.status_code == 200 assert response.reason == "OK" @@ -63,19 +63,19 @@ def test_response_html(send_request): def test_raise_for_status(client): response = client.send_request( - HttpRequest("GET", "http://localhost:5000/basic/string"), + HttpRequest("GET", "/basic/string"), ) response.raise_for_status() response = client.send_request( - HttpRequest("GET", "http://localhost:5000/errors/403"), + HttpRequest("GET", "/errors/403"), ) assert response.status_code == 403 with pytest.raises(HttpResponseError): response.raise_for_status() response = client.send_request( - HttpRequest("GET", "http://localhost:5000/errors/500"), + HttpRequest("GET", "/errors/500"), retry_total=0, # takes too long with retires on 500 ) assert response.status_code == 500 @@ -84,7 +84,7 @@ def test_raise_for_status(client): def test_response_repr(send_request): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/string") + request=HttpRequest("GET", "/basic/string") ) assert repr(response) == "" @@ -93,7 +93,7 @@ def test_response_content_type_encoding(send_request): Use the charset encoding in the Content-Type header if possible. """ response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1") + request=HttpRequest("GET", "/encoding/latin-1") ) assert response.content_type == "text/plain; charset=latin-1" assert response.text == u"Latin 1: ÿ" @@ -105,7 +105,7 @@ def test_response_autodetect_encoding(send_request): Autodetect encoding if there is no Content-Type header. """ response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1") + request=HttpRequest("GET", "/encoding/latin-1") ) assert response.text == u'Latin 1: ÿ' @@ -118,7 +118,7 @@ def test_response_fallback_to_autodetect(send_request): Fallback to autodetection if we get an invalid charset in the Content-Type header. """ response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/invalid-codec-name") + request=HttpRequest("GET", "/encoding/invalid-codec-name") ) assert response.headers["Content-Type"] == "text/plain; charset=invalid-codec-name" @@ -132,7 +132,7 @@ def test_response_no_charset_with_ascii_content(send_request): even with no charset specified. """ response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/no-charset"), + request=HttpRequest("GET", "/encoding/no-charset"), ) assert response.headers["Content-Type"] == "text/plain" @@ -147,7 +147,7 @@ def test_response_no_charset_with_iso_8859_1_content(send_request): even with no charset specified. """ response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/iso-8859-1"), + request=HttpRequest("GET", "/encoding/iso-8859-1"), ) assert response.text == u"Accented: Österreich" assert response.encoding is None @@ -155,7 +155,7 @@ def test_response_no_charset_with_iso_8859_1_content(send_request): def test_response_set_explicit_encoding(send_request): # Deliberately incorrect charset response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/latin-1-with-utf-8"), + request=HttpRequest("GET", "/encoding/latin-1-with-utf-8"), ) assert response.headers["Content-Type"] == "text/plain; charset=utf-8" response.encoding = "latin-1" @@ -164,33 +164,33 @@ def test_response_set_explicit_encoding(send_request): def test_json(send_request): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/basic/json"), + request=HttpRequest("GET", "/basic/json"), ) assert response.json() == {"greeting": "hello", "recipient": "world"} assert response.encoding is None def test_json_with_specified_encoding(send_request): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/json"), + request=HttpRequest("GET", "/encoding/json"), ) assert response.json() == {"greeting": "hello", "recipient": "world"} assert response.encoding == "utf-16" def test_emoji(send_request): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/emoji"), + request=HttpRequest("GET", "/encoding/emoji"), ) assert response.text == u"👩" def test_emoji_family_with_skin_tone_modifier(send_request): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/emoji-family-skin-tone-modifier"), + request=HttpRequest("GET", "/encoding/emoji-family-skin-tone-modifier"), ) assert response.text == u"👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987" def test_korean_nfc(send_request): response = send_request( - request=HttpRequest("GET", "http://localhost:5000/encoding/korean"), + request=HttpRequest("GET", "/encoding/korean"), ) assert response.text == u"아가" @@ -198,7 +198,7 @@ def test_urlencoded_content(send_request): send_request( request=HttpRequest( "POST", - "http://localhost:5000/urlencoded/pet/add/1", + "/urlencoded/pet/add/1", data={ "pet_type": "dog", "pet_food": "meat", "name": "Fido", "pet_age": 42 } ), ) @@ -206,7 +206,7 @@ def test_urlencoded_content(send_request): def test_multipart_files_content(send_request): request = HttpRequest( "POST", - "http://localhost:5000/multipart/basic", + "/multipart/basic", files={"fileContent": io.BytesIO(b"")}, ) send_request(request) @@ -214,7 +214,7 @@ def test_multipart_files_content(send_request): def test_multipart_data_and_files_content(send_request): request = HttpRequest( "POST", - "http://localhost:5000/multipart/data-and-files", + "/multipart/data-and-files", data={"message": "Hello, world!"}, files={"fileContent": io.BytesIO(b"")}, ) @@ -243,7 +243,7 @@ def data(): files = {"file": fileobj} request = HttpRequest( "POST", - "http://localhost:5000/multipart/non-seekable-filelike", + "/multipart/non-seekable-filelike", files=files, ) send_request(request) @@ -251,7 +251,7 @@ def data(): def test_get_xml_basic(send_request): request = HttpRequest( "GET", - "http://localhost:5000/xml/basic", + "/xml/basic", ) response = send_request(request) parsed_xml = ET.fromstring(response.text) @@ -281,7 +281,7 @@ def test_put_xml_basic(send_request): request = HttpRequest( "PUT", - "http://localhost:5000/xml/basic", + "/xml/basic", content=ET.fromstring(basic_body), ) send_request(request) @@ -298,7 +298,7 @@ def _convert(self): def test_request_no_conversion(send_request): - request = MockHttpRequest("GET", "http://localhost:5000/basic/string") + request = MockHttpRequest("GET", "/basic/string") response = send_request( request=request, ) diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py index c8eee614ea6f..33d26cd654d7 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py @@ -23,7 +23,7 @@ def _assert_stream_state(response, open): assert all(checks) def test_iter_raw(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: raw = b"" for part in response.iter_raw(): @@ -37,7 +37,7 @@ def test_iter_raw(client): assert response.is_stream_consumed def test_iter_raw_on_iterable(client): - request = HttpRequest("GET", "http://localhost:5000/streams/iterable") + request = HttpRequest("GET", "/streams/iterable") with client.send_request(request, stream=True) as response: raw = b"" @@ -46,7 +46,7 @@ def test_iter_raw_on_iterable(client): assert raw == b"Hello, world!" def test_iter_with_error(client): - request = HttpRequest("GET", "http://localhost:5000/errors/403") + request = HttpRequest("GET", "/errors/403") with client.send_request(request, stream=True) as response: with pytest.raises(HttpResponseError): @@ -65,7 +65,7 @@ def test_iter_with_error(client): assert response.is_closed def test_iter_raw_with_chunksize(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: parts = [part for part in response.iter_raw(chunk_size=5)] @@ -80,7 +80,7 @@ def test_iter_raw_with_chunksize(client): assert parts == [b"Hello, world!"] def test_iter_raw_num_bytes_downloaded(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: num_downloaded = response.num_bytes_downloaded @@ -89,7 +89,7 @@ def test_iter_raw_num_bytes_downloaded(client): num_downloaded = response.num_bytes_downloaded def test_iter_bytes(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: raw = b"" @@ -104,7 +104,7 @@ def test_iter_bytes(client): assert raw == b"Hello, world!" def test_iter_bytes_with_chunk_size(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: parts = [part for part in response.iter_bytes(chunk_size=5)] @@ -119,7 +119,7 @@ def test_iter_bytes_with_chunk_size(client): assert parts == [b"Hello, world!"] def test_iter_text(client): - request = HttpRequest("GET", "http://localhost:5000/basic/string") + request = HttpRequest("GET", "/basic/string") with client.send_request(request, stream=True) as response: content = "" @@ -128,7 +128,7 @@ def test_iter_text(client): assert content == "Hello, world!" def test_iter_text_with_chunk_size(client): - request = HttpRequest("GET", "http://localhost:5000/basic/string") + request = HttpRequest("GET", "/basic/string") with client.send_request(request, stream=True) as response: parts = [part for part in response.iter_text(chunk_size=5)] @@ -143,7 +143,7 @@ def test_iter_text_with_chunk_size(client): assert parts == ["Hello, world!"] def test_iter_lines(client): - request = HttpRequest("GET", "http://localhost:5000/basic/lines") + request = HttpRequest("GET", "/basic/lines") with client.send_request(request, stream=True) as response: content = [] @@ -152,7 +152,7 @@ def test_iter_lines(client): assert content == ["Hello,\n", "world!"] def test_sync_streaming_response(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: assert response.status_code == 200 @@ -165,7 +165,7 @@ def test_sync_streaming_response(client): assert response.is_closed def test_cannot_read_after_stream_consumed(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: content = b"" @@ -180,13 +180,13 @@ def test_cannot_read_after_stream_consumed(client): assert "You are attempting to read or stream content that has already been streamed." in str(ex.value) def test_cannot_read_after_response_closed(client): - request = HttpRequest("GET", "http://localhost:5000/streams/basic") + request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: response.close() with pytest.raises(StreamClosedError) as ex: response.read() - assert "You can not try to read or stream this response's content, since the response has been closed" in str(ex.value) + assert "You can not try to read or stream this response's content, since the response's stream has been closed" in str(ex.value) def test_decompress_plain_no_header(client): # thanks to Xiang Yan for this test! @@ -232,7 +232,7 @@ def test_decompress_compressed_header(client): def test_iter_read(client): # thanks to McCoy Patiño for this test! - request = HttpRequest("GET", "http://localhost:5000/basic/lines") + request = HttpRequest("GET", "/basic/lines") response = client.send_request(request, stream=True) response.read() iterator = response.iter_lines() @@ -247,7 +247,7 @@ def test_iter_read_back_and_forth(client): # the reason why the code flow is like this, is because the 'iter_x' functions don't # actually read the contents into the response, the output them. Once they're yielded, # the stream is closed, so you have to catch the output when you iterate through it - request = HttpRequest("GET", "http://localhost:5000/basic/lines") + request = HttpRequest("GET", "/basic/lines") response = client.send_request(request, stream=True) iterator = response.iter_lines() for line in iterator: diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py deleted file mode 100644 index 623033080bd1..000000000000 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py +++ /dev/null @@ -1,37 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) Microsoft Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -# -------------------------------------------------------------------------- -import pytest -from azure.core.pipeline.transport import HttpRequest, AioHttpTransport -"""This file does a simple call to the testserver to make sure we can use the testserver""" - -@pytest.mark.asyncio -async def test_smoke(port): - request = HttpRequest(method="GET", url="http://localhost:{}/basic/string".format(port)) - async with AioHttpTransport() as sender: - response = await sender.send(request) - response.raise_for_status() - await response.load_body() - assert response.text() == "Hello, world!" \ No newline at end of file From 911230c151ee5c32dc339951b99c0adeb9e5f16a Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Sat, 26 Jun 2021 18:01:35 -0400 Subject: [PATCH 44/64] fix tests and bump version --- .../azure/core/_pipeline_client_async.py | 4 +-- sdk/core/azure-core/azure/core/_version.py | 2 +- .../azure-core/azure/core/pipeline/_tools.py | 2 +- .../azure/core/pipeline/_tools_async.py | 31 +++++++++++++------ .../azure/core/pipeline/transport/_aiohttp.py | 25 +++------------ .../pipeline/transport/_requests_asyncio.py | 25 +++------------ .../core/pipeline/transport/_requests_trio.py | 25 +++------------ .../azure-core/azure/core/rest/_rest_py3.py | 5 ++- .../coretestserver/test_routes/basic.py | 6 ++-- .../coretestserver/test_routes/encoding.py | 12 +++---- .../test_rest_context_manager.py | 6 +--- .../test_rest_http_request.py | 1 - .../test_rest_http_response.py | 3 +- 13 files changed, 54 insertions(+), 93 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index 5916ba433673..64b53d746b1b 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -202,8 +202,8 @@ def send_request( ) -> Awaitable[AsyncHttpResponse]: """**Provisional** method that runs the network request through the client's chained policies. - This method is marked as **provisional**, meaning it can be changed - + This method is marked as **provisional**, meaning it can be changed. + :param request: The network request you want to make. Required. :type request: ~azure.core.rest.HttpRequest :keyword bool stream: Whether the response payload will be streamed. Defaults to False. diff --git a/sdk/core/azure-core/azure/core/_version.py b/sdk/core/azure-core/azure/core/_version.py index d7a104234f8f..48bb9d819b66 100644 --- a/sdk/core/azure-core/azure/core/_version.py +++ b/sdk/core/azure-core/azure/core/_version.py @@ -9,4 +9,4 @@ # regenerated. # -------------------------------------------------------------------------- -VERSION = "1.15.1" +VERSION = "1.16.0" diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools.py b/sdk/core/azure-core/azure/core/pipeline/_tools.py index 6653baf72b2a..2ff45beb2bd2 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools.py @@ -23,9 +23,9 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +from typing import TYPE_CHECKING from ..exceptions import StreamClosedError, StreamConsumedError -from typing import TYPE_CHECKING if TYPE_CHECKING: from typing import ( diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index 8af2e9266cea..3ffbeafa9c1d 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -53,26 +53,37 @@ def _stream_download_helper( decompress=decompress, ) -def iter_bytes_helper( +async def iter_bytes_helper( stream_download_generator: Callable, response, chunk_size: Optional[int] = None, ) -> AsyncIterator[bytes]: - return _stream_download_helper( - decompress=True, - stream_download_generator=stream_download_generator, - response=response, - chunk_size=chunk_size - ) + content = response._get_content() # pylint: disable=protected-access + if content is not None: + if chunk_size is None: + chunk_size = len(content) + for i in range(0, len(content), chunk_size): + yield content[i: i + chunk_size] + else: + async for part in _stream_download_helper( + decompress=True, + stream_download_generator=stream_download_generator, + response=response, + chunk_size=chunk_size + ): + response._num_bytes_downloaded += len(part) + yield part -def iter_raw_helper( +async def iter_raw_helper( stream_download_generator: Callable, response, chunk_size: Optional[int] = None ) -> AsyncIterator[bytes]: - return _stream_download_helper( + async for part in _stream_download_helper( decompress=False, stream_download_generator=stream_download_generator, response=response, chunk_size=chunk_size - ) + ): + response._num_bytes_downloaded += len(part) + yield part diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 9e41634602a3..242d829594b5 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -419,39 +419,24 @@ def text(self) -> str: async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_raw_helper( - stream_download_generator=AioHttpStreamDownloadGenerator, - response=self, - chunk_size=chunk_size, - ): - self._num_bytes_downloaded += len(part) + async for part in iter_raw_helper(AioHttpStreamDownloadGenerator, self, chunk_size): yield part await self.close() async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - content = self._get_content() # pylint: disable=protected-access - if content is not None: - if chunk_size is None: - chunk_size = len(content) - for i in range(0, len(content), chunk_size): - yield content[i: i + chunk_size] - else: - async for part in iter_bytes_helper( - stream_download_generator=AioHttpStreamDownloadGenerator, - response=self, - chunk_size=chunk_size - ): - self._num_bytes_downloaded += len(part) - yield part + async for part in iter_bytes_helper(AioHttpStreamDownloadGenerator, self, chunk_size): + yield part await self.close() def __getstate__(self): diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index 68ae1e72f382..0ce465119d7b 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -202,37 +202,22 @@ class RestAsyncioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsT async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_raw_helper( - stream_download_generator=AsyncioStreamDownloadGenerator, - response=self, - chunk_size=chunk_size, - ): - self._num_bytes_downloaded += len(part) + async for part in iter_raw_helper(AsyncioRequestsTransportResponse, self, chunk_size): yield part await self.close() async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - content = self._get_content() # pylint: disable=protected-access - if content is not None: - if chunk_size is None: - chunk_size = len(content) - for i in range(0, len(content), chunk_size): - yield content[i: i + chunk_size] - else: - async for part in iter_bytes_helper( - stream_download_generator=AsyncioStreamDownloadGenerator, - response=self, - chunk_size=chunk_size - ): - self._num_bytes_downloaded += len(part) - yield part + async for part in iter_bytes_helper(AsyncioRequestsTransportResponse, self, chunk_size): + yield part await self.close() diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index 2b1ee3b60c4e..28a1ae757a0e 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -121,39 +121,24 @@ class RestTrioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsTran """ async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_raw_helper( - stream_download_generator=TrioStreamDownloadGenerator, - response=self, - chunk_size=chunk_size, - ): - self._num_bytes_downloaded += len(part) + async for part in iter_raw_helper(TrioStreamDownloadGenerator, self, chunk_size): yield part await self.close() async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process + :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - content = self._get_content() # pylint: disable=protected-access - if content is not None: - if chunk_size is None: - chunk_size = len(content) - for i in range(0, len(content), chunk_size): - yield content[i: i + chunk_size] - else: - async for part in iter_bytes_helper( - stream_download_generator=TrioStreamDownloadGenerator, - response=self, - chunk_size=chunk_size - ): - self._num_bytes_downloaded += len(part) - yield part + async for part in iter_bytes_helper(TrioStreamDownloadGenerator, self, chunk_size): + yield part await self.close() async def close(self) -> None: diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index 61f573394deb..90f0e8c011a6 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -39,7 +39,6 @@ Type, Union, ) -from abc import abstractmethod from azure.core.exceptions import HttpResponseError @@ -477,7 +476,7 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: yield _ raise NotImplementedError() - async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: + async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: # pylint: disable=unused-argument """Asynchronously iterates over the response's bytes. Will decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. @@ -492,7 +491,7 @@ async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: yield _ raise NotImplementedError() - async def iter_text(self, chunk_size: int = None) -> AsyncIterator[str]: + async def iter_text(self, chunk_size: int = None) -> AsyncIterator[str]: # pylint: disable=unused-argument """Asynchronously iterates over the text in the response. :param int chunk_size: The maximum size of each chunk iterated over. diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py index 4b7d5ae92ad4..0f0735522d11 100644 --- a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py @@ -1,4 +1,4 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- # ------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See LICENSE.txt in the project root for @@ -56,9 +56,9 @@ def complicated_json(): assert request.json['SpacesAfterUnicode'] == 'Text ' assert request.json['SpacesBeforeAndAfterByte'] == ' Text ' assert request.json['SpacesBeforeAndAfterUnicode'] == ' Text ' - assert request.json['啊齄丂狛'] == 'ꀕ' + assert request.json[u'啊齄丂狛'] == u'ꀕ' assert request.json['RowKey'] == 'test2' - assert request.json['啊齄丂狛狜'] == 'hello' + assert request.json[u'啊齄丂狛狜'] == 'hello' assert request.json["singlequote"] == "a''''b" assert request.json["doublequote"] == 'a""""b' assert request.json["None"] == None diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py index 12224e568ee5..104ef7608bd0 100644 --- a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py @@ -1,4 +1,4 @@ -# coding: utf-8 +# -*- coding: utf-8 -*- # ------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See LICENSE.txt in the project root for @@ -15,7 +15,7 @@ @encoding_api.route('/latin-1', methods=['GET']) def latin_1(): r = Response( - "Latin 1: ÿ".encode("latin-1"), status=200 + u"Latin 1: ÿ".encode("latin-1"), status=200 ) r.headers["Content-Type"] = "text/plain; charset=latin-1" return r @@ -23,7 +23,7 @@ def latin_1(): @encoding_api.route('/latin-1-with-utf-8', methods=['GET']) def latin_1_charset_utf8(): r = Response( - "Latin 1: ÿ".encode("latin-1"), status=200 + u"Latin 1: ÿ".encode("latin-1"), status=200 ) r.headers["Content-Type"] = "text/plain; charset=utf-8" return r @@ -39,7 +39,7 @@ def latin_1_no_charset(): @encoding_api.route('/iso-8859-1', methods=['GET']) def iso_8859_1(): r = Response( - "Accented: Österreich".encode("iso-8859-1"), status=200 + u"Accented: Österreich".encode("iso-8859-1"), status=200 ) r.headers["Content-Type"] = "text/plain" return r @@ -47,14 +47,14 @@ def iso_8859_1(): @encoding_api.route('/emoji', methods=['GET']) def emoji(): r = Response( - "👩", status=200 + u"👩", status=200 ) return r @encoding_api.route('/emoji-family-skin-tone-modifier', methods=['GET']) def emoji_family_skin_tone_modifier(): r = Response( - "👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987", status=200 + u"👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987", status=200 ) return r diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py index c705fe6045da..34f802971537 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_context_manager.py @@ -5,12 +5,8 @@ # license information. # ------------------------------------------------------------------------- import pytest -import mock from azure.core.rest import HttpRequest -from azure.core.exceptions import HttpResponseError, ResponseNotReadError - -from azure.core.pipeline import Pipeline, transport -from azure.core.pipeline.transport import RequestsTransport +from azure.core.exceptions import ResponseNotReadError def test_normal_call(client, port): def _raise_and_get_text(response): diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py index 74d815648589..a41a911c3a6d 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_request.py @@ -253,7 +253,6 @@ def content(): # in this case, request._data is what we end up passing to the requests transport assert isinstance(request._data, collections.Iterable) - def test_complicated_json(client): # thanks to Sean Kane for this test! input = { diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py index 5915e5548c63..d31d54522a78 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py @@ -1,4 +1,5 @@ -# coding: utf-8 +#!/usr/bin/env python +# -*- coding: utf-8 -*- # ------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See LICENSE.txt in the project root for From a43004c9c56102acdf4638f2ac1ef148c6fe92a0 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Sun, 27 Jun 2021 16:50:42 -0400 Subject: [PATCH 45/64] mypy --- sdk/core/azure-core/azure/core/rest/_helpers.py | 11 +++++++---- sdk/core/azure-core/azure/core/rest/_rest.py | 6 +----- sdk/core/azure-core/azure/core/rest/_rest_py3.py | 6 +----- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/sdk/core/azure-core/azure/core/rest/_helpers.py b/sdk/core/azure-core/azure/core/rest/_helpers.py index 853acb4724e4..fa309dabc097 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers.py @@ -118,8 +118,9 @@ def _format_data(data): return (data_name, data, "application/octet-stream") return (None, cast(str, data)) -def set_urlencoded_body(data): +def set_urlencoded_body(data, has_files): body = {} + default_headers = {} for f, d in data.items(): if not d: continue @@ -129,9 +130,11 @@ def set_urlencoded_body(data): else: _verify_data_object(f, d) body[f] = d - return { - "Content-Type": "application/x-www-form-urlencoded" - }, body + if not has_files: + # little hacky, but for files we don't send a content type with + # boundary so requests / aiohttp etc deal with it + default_headers["Content-Type"] = "application/x-www-form-urlencoded" + return default_headers, body def set_multipart_body(files): formatted_files = { diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index 2b5131179e83..6d98480ce58b 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -137,11 +137,7 @@ def _set_body(self, content, data, files, json): if files: default_headers, self._files = set_multipart_body(files) if data: - default_headers, self._data = set_urlencoded_body(data) - if files and data: - # little hacky, but for files we don't send a content type with - # boundary so requests / aiohttp etc deal with it - default_headers.pop("Content-Type") + default_headers, self._data = set_urlencoded_body(data, bool(files)) return default_headers def _update_headers(self, default_headers): diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index 90f0e8c011a6..aa8295724468 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -172,11 +172,7 @@ def _set_body( if files: default_headers, self._files = set_multipart_body(files) if data: - default_headers, self._data = set_urlencoded_body(data) - if files and data: - # little hacky, but for files we don't send a content type with - # boundary so requests / aiohttp etc deal with it - default_headers.pop("Content-Type") + default_headers, self._data = set_urlencoded_body(data, has_files=bool(files)) return default_headers @property From b392f7c9dfd262fef46cd07a6ff93c3da0bd2821 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 09:16:01 -0400 Subject: [PATCH 46/64] split up async chunksize tests --- .../test_rest_stream_responses_async.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py index 15587805c211..163135dad3af 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py @@ -56,7 +56,7 @@ async def test_iter_with_error(client): assert response.is_closed @pytest.mark.asyncio -async def test_iter_raw_with_chunksize(client): +async def test_iter_raw_with_chunksize_5(client): request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: @@ -65,12 +65,18 @@ async def test_iter_raw_with_chunksize(client): parts.append(part) assert parts == [b'Hello', b', wor', b'ld!'] +@pytest.mark.asyncio +async def test_iter_raw_with_chunksize_13(client): + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: parts = [] async for part in response.iter_raw(chunk_size=13): parts.append(part) assert parts == [b"Hello, world!"] +@pytest.mark.asyncio +async def test_iter_raw_with_chunksize_20(client): + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: parts = [] async for part in response.iter_raw(chunk_size=20): @@ -102,7 +108,7 @@ async def test_iter_bytes(client): assert raw == b"Hello, world!" @pytest.mark.asyncio -async def test_iter_bytes_with_chunk_size(client): +async def test_iter_bytes_with_chunk_size_5(client): request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: @@ -111,12 +117,18 @@ async def test_iter_bytes_with_chunk_size(client): parts.append(part) assert parts == [b"Hello", b", wor", b"ld!"] +@pytest.mark.asyncio +async def test_iter_bytes_with_chunk_size_13(client): + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: parts = [] async for part in response.iter_bytes(chunk_size=13): parts.append(part) assert parts == [b"Hello, world!"] +@pytest.mark.asyncio +async def test_iter_bytes_with_chunk_size_20(client): + request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: parts = [] async for part in response.iter_bytes(chunk_size=20): From 61edc06ee6f1b1c88da2d177d2673669a2a9bffd Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 10:00:10 -0400 Subject: [PATCH 47/64] pylint --- sdk/core/azure-core/azure/core/rest/_rest_py3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index aa8295724468..791f336d95ee 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -457,7 +457,7 @@ async def read(self) -> bytes: self._set_content(b"".join(parts)) return self._get_content() - async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: + async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # pylint: disable=unused-argument """Asynchronously iterates over the response's bytes. Will not decompress in the process :param int chunk_size: The maximum size of each chunk iterated over. From 038f8dac7718ca7797ab9474fcf922141484238f Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 10:45:30 -0400 Subject: [PATCH 48/64] make exit call close --- sdk/core/azure-core/azure/core/rest/_rest_py3.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index 791f336d95ee..059763b690f9 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -355,8 +355,7 @@ def close(self) -> None: self.internal_response.close() def __exit__(self, *args) -> None: - self.is_closed = True - self.internal_response.__exit__(*args) + self.close() def read(self) -> bytes: """Read the response's bytes. @@ -521,8 +520,7 @@ async def close(self) -> None: await asyncio.sleep(0) async def __aexit__(self, *args) -> None: - self.is_closed = True - await self.internal_response.__aexit__(*args) + await self.close() def __repr__(self) -> str: content_type_str = ( From 49e3c33fe7c14ae249f1ba7743d9fdd83df1ec79 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 11:49:50 -0400 Subject: [PATCH 49/64] remove chunk_size --- .../azure-core/azure/core/pipeline/_tools.py | 32 +++----- .../azure/core/pipeline/_tools_async.py | 14 +--- .../azure/core/pipeline/transport/_aiohttp.py | 17 ++-- .../pipeline/transport/_requests_asyncio.py | 12 ++- .../pipeline/transport/_requests_basic.py | 22 ++++-- .../core/pipeline/transport/_requests_trio.py | 12 ++- sdk/core/azure-core/azure/core/rest/_rest.py | 20 ++--- .../azure-core/azure/core/rest/_rest_py3.py | 33 +++----- .../test_rest_stream_responses_async.py | 78 ------------------- .../test_rest_stream_responses.py | 45 ----------- 10 files changed, 67 insertions(+), 218 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools.py b/sdk/core/azure-core/azure/core/pipeline/_tools.py index 2ff45beb2bd2..eae6bcfa4437 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools.py @@ -44,8 +44,8 @@ def await_result(func, *args, **kwargs): ) return result -def _stream_download_helper(decompress, stream_download_generator, response, chunk_size=None): - # type: (bool, Callable, HttpResponse, Optional[int]) -> Iterator[bytes] +def _stream_download_helper(decompress, stream_download_generator, response): + # type: (bool, Callable, HttpResponse) -> Iterator[bytes] if response.is_stream_consumed: raise StreamConsumedError() if response.is_closed: @@ -55,37 +55,31 @@ def _stream_download_helper(decompress, stream_download_generator, response, chu stream_download = stream_download_generator( pipeline=None, response=response, - chunk_size=chunk_size or response._connection_data_block_size, # pylint: disable=protected-access decompress=decompress, ) for part in stream_download: response._num_bytes_downloaded += len(part) yield part -def iter_bytes_helper(stream_download_generator, response, chunk_size=None): - # type: (Callable, HttpResponse, Optional[int]) -> Iterator[bytes] +def iter_bytes_helper(stream_download_generator, response): + # type: (Callable, HttpResponse) -> Iterator[bytes] if response._has_content(): # pylint: disable=protected-access - if chunk_size is None: - chunk_size = len(response.content) - for i in range(0, len(response.content), chunk_size): - yield response.content[i: i + chunk_size] + yield response._get_content() # pylint: disable=protected-access else: for part in _stream_download_helper( decompress=True, stream_download_generator=stream_download_generator, response=response, - chunk_size=chunk_size ): yield part response.close() -def iter_raw_helper(stream_download_generator, response, chunk_size=None): - # type: (Callable, HttpResponse, Optional[int]) -> Iterator[bytes] +def iter_raw_helper(stream_download_generator, response): + # type: (Callable, HttpResponse) -> Iterator[bytes] for raw_bytes in _stream_download_helper( decompress=False, stream_download_generator=stream_download_generator, response=response, - chunk_size=chunk_size ): yield raw_bytes response.close() @@ -98,11 +92,7 @@ def to_rest_response_helper(pipeline_transport_response, response_type): response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access return response -def set_block_size(response, **kwargs): - chunk_size = kwargs.pop("chunk_size", None) - if not chunk_size: - if hasattr(response, "block_size"): - chunk_size = response.block_size - elif hasattr(response, "_connection_data_block_size"): - chunk_size = response._connection_data_block_size # pylint: disable=protected-access - return chunk_size +def set_block_size(response): + if hasattr(response, "block_size"): + return response.block_size + return response._connection_data_block_size # pylint: disable=protected-access diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index 3ffbeafa9c1d..a2b57546a624 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -38,7 +38,6 @@ def _stream_download_helper( decompress: bool, stream_download_generator: Callable, response, - chunk_size: Optional[int] = None, ) -> AsyncIterator[bytes]: if response.is_stream_consumed: raise StreamConsumedError() @@ -49,27 +48,20 @@ def _stream_download_helper( return stream_download_generator( pipeline=None, response=response, - chunk_size=chunk_size, decompress=decompress, ) async def iter_bytes_helper( stream_download_generator: Callable, response, - chunk_size: Optional[int] = None, ) -> AsyncIterator[bytes]: - content = response._get_content() # pylint: disable=protected-access - if content is not None: - if chunk_size is None: - chunk_size = len(content) - for i in range(0, len(content), chunk_size): - yield content[i: i + chunk_size] + if response._has_content(): # pylint: disable=protected-access + yield response._get_content() # pylint: disable=protected-access else: async for part in _stream_download_helper( decompress=True, stream_download_generator=stream_download_generator, response=response, - chunk_size=chunk_size ): response._num_bytes_downloaded += len(part) yield part @@ -77,13 +69,11 @@ async def iter_bytes_helper( async def iter_raw_helper( stream_download_generator: Callable, response, - chunk_size: Optional[int] = None ) -> AsyncIterator[bytes]: async for part in _stream_download_helper( decompress=False, stream_download_generator=stream_download_generator, response=response, - chunk_size=chunk_size ): response._num_bytes_downloaded += len(part) yield part diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 242d829594b5..979e85638716 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -220,11 +220,11 @@ class AioHttpStreamDownloadGenerator(AsyncIterator): :param bool decompress: If True which is default, will attempt to decode the body based on the *content-encoding* header. """ - def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, *, decompress=True, **kwargs) -> None: + def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, *, decompress=True) -> None: self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = set_block_size(response, chunk_size=kwargs.pop("chunk_size", None), **kwargs) + self.block_size = set_block_size(response) self._decompress = decompress self.content_length = int(response.internal_response.headers.get('Content-Length', 0)) self._decompressor = None @@ -377,9 +377,8 @@ def __init__( *, request: RestHttpRequest, internal_response, - **kwargs ): - super().__init__(request=request, internal_response=internal_response, **kwargs) + super().__init__(request=request, internal_response=internal_response) self.status_code = internal_response.status self.headers = CIMultiDict(internal_response.headers) # type: ignore self.reason = internal_response.reason @@ -417,25 +416,23 @@ def text(self) -> str: return content.decode(encoding) - async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: + async def iter_raw(self) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_raw_helper(AioHttpStreamDownloadGenerator, self, chunk_size): + async for part in iter_raw_helper(AioHttpStreamDownloadGenerator, self): yield part await self.close() - async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: + async def iter_bytes(self) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_bytes_helper(AioHttpStreamDownloadGenerator, self, chunk_size): + async for part in iter_bytes_helper(AioHttpStreamDownloadGenerator, self): yield part await self.close() diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index 0ce465119d7b..a05e39753b3d 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -151,7 +151,7 @@ def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, **kwargs) -> self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = set_block_size(response, chunk_size=kwargs.pop("chunk_size", None), **kwargs) + self.block_size = set_block_size(response) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) @@ -200,24 +200,22 @@ class RestAsyncioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsT """Asynchronous streaming of data from the response. """ - async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: + async def iter_raw(self) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_raw_helper(AsyncioRequestsTransportResponse, self, chunk_size): + async for part in iter_raw_helper(AsyncioRequestsTransportResponse, self): yield part await self.close() - async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: + async def iter_bytes(self) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_bytes_helper(AsyncioRequestsTransportResponse, self, chunk_size): + async for part in iter_bytes_helper(AsyncioRequestsTransportResponse, self): yield part await self.close() diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py index 50e5cf1fefe2..0d7f50f3fda1 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py @@ -140,7 +140,7 @@ def __init__(self, pipeline, response, **kwargs): self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = set_block_size(response, chunk_size=kwargs.pop("chunk_size", None), **kwargs) + self.block_size = set_block_size(response) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) @@ -239,20 +239,28 @@ def _to_rest_response(self): class RestRequestsTransportResponse(RestHttpResponse, _RestRequestsTransportResponseBase): - def iter_bytes(self, chunk_size=None): - # type: (Optional[int]) -> Iterator[bytes] + def iter_bytes(self): + # type: () -> Iterator[bytes] + """Iterates over the response's bytes. Will decompress in the process + + :return: An iterator of bytes from the response + :rtype: Iterator[str] + """ return iter_bytes_helper( stream_download_generator=StreamDownloadGenerator, response=self, - chunk_size=chunk_size, ) - def iter_raw(self, chunk_size=None): - # type: (Optional[int]) -> Iterator[bytes] + def iter_raw(self): + # type: () -> Iterator[bytes] + """Iterates over the response's bytes. Will not decompress in the process + + :return: An iterator of bytes from the response + :rtype: Iterator[str] + """ return iter_raw_helper( stream_download_generator=StreamDownloadGenerator, response=self, - chunk_size=chunk_size, ) class RequestsTransport(HttpTransport): diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index 28a1ae757a0e..0a00a7f12988 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -67,7 +67,7 @@ def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, **kwargs) -> self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = set_block_size(response, chunk_size=kwargs.pop("chunk_size", None), **kwargs) + self.block_size = set_block_size(response) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) @@ -119,25 +119,23 @@ def _to_rest_response(self): class RestTrioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore """Asynchronous streaming of data from the response. """ - async def iter_raw(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: + async def iter_raw(self) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_raw_helper(TrioStreamDownloadGenerator, self, chunk_size): + async for part in iter_raw_helper(TrioStreamDownloadGenerator, self): yield part await self.close() - async def iter_bytes(self, chunk_size: int = None) -> AsyncIteratorType[bytes]: + async def iter_bytes(self) -> AsyncIteratorType[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_bytes_helper(TrioStreamDownloadGenerator, self, chunk_size): + async for part in iter_bytes_helper(TrioStreamDownloadGenerator, self): yield part await self.close() diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index 6d98480ce58b..fc434014ffbc 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -352,29 +352,29 @@ def read(self): self._set_content(b"".join(self.iter_bytes())) return self.content - def iter_raw(self, chunk_size=None): - # type: (Optional[int]) -> Iterator[bytes] + def iter_raw(self): + # type: () -> Iterator[bytes] """Iterate over the raw response bytes """ raise NotImplementedError() - def iter_bytes(self, chunk_size=None): - # type: (Optional[int]) -> Iterator[bytes] + def iter_bytes(self): + # type: () -> Iterator[bytes] """Iterate over the response bytes """ raise NotImplementedError() - def iter_text(self, chunk_size=None): - # type: (int) -> Iterator[str] + def iter_text(self): + # type: () -> Iterator[str] """Iterate over the response text """ - for byte in self.iter_bytes(chunk_size): + for byte in self.iter_bytes(): text = byte.decode(self.encoding or "utf-8") yield text - def iter_lines(self, chunk_size=None): - # type: (int) -> Iterator[str] - for text in self.iter_text(chunk_size): + def iter_lines(self): + # type: () -> Iterator[str] + for text in self.iter_text(): lines = parse_lines_from_text(text) for line in lines: yield line diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index 059763b690f9..3c2296e5978e 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -216,7 +216,6 @@ def __init__( *, request: HttpRequest, internal_response, - **kwargs # pylint: disable=unused-argument ): self.request = request self.internal_response = internal_response @@ -367,43 +366,39 @@ def read(self) -> bytes: self._set_content(b"".join(self.iter_bytes())) return self.content - def iter_raw(self, chunk_size: Optional[int] = None) -> Iterator[bytes]: + def iter_raw(self) -> Iterator[bytes]: """Iterates over the response's bytes. Will not decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An iterator of bytes from the response :rtype: Iterator[str] """ raise NotImplementedError() - def iter_bytes(self, chunk_size: Optional[int] = None) -> Iterator[bytes]: + def iter_bytes(self) -> Iterator[bytes]: """Iterates over the response's bytes. Will decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An iterator of bytes from the response :rtype: Iterator[str] """ raise NotImplementedError() - def iter_text(self, chunk_size: int = None) -> Iterator[str]: + def iter_text(self) -> Iterator[str]: """Iterates over the text in the response. - :param int chunk_size: The maximum size of each chunk iterated over. :return: An iterator of string. Each string chunk will be a text from the response :rtype: Iterator[str] """ - for byte in self.iter_bytes(chunk_size): + for byte in self.iter_bytes(): text = byte.decode(self.encoding or "utf-8") yield text - def iter_lines(self, chunk_size: int = None) -> Iterator[str]: + def iter_lines(self) -> Iterator[str]: """Iterates over the lines in the response. - :param int chunk_size: The maximum size of each chunk iterated over. :return: An iterator of string. Each string chunk will be a line from the response :rtype: Iterator[str] """ - for text in self.iter_text(chunk_size): + for text in self.iter_text(): lines = parse_lines_from_text(text) for line in lines: yield line @@ -456,10 +451,9 @@ async def read(self) -> bytes: self._set_content(b"".join(parts)) return self._get_content() - async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # pylint: disable=unused-argument + async def iter_raw(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ @@ -471,10 +465,9 @@ async def iter_raw(self, chunk_size: int = None) -> AsyncIterator[bytes]: # pyl yield _ raise NotImplementedError() - async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: # pylint: disable=unused-argument + async def iter_bytes(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ @@ -486,25 +479,23 @@ async def iter_bytes(self, chunk_size: int = None) -> AsyncIterator[bytes]: # p yield _ raise NotImplementedError() - async def iter_text(self, chunk_size: int = None) -> AsyncIterator[str]: # pylint: disable=unused-argument + async def iter_text(self) -> AsyncIterator[str]: """Asynchronously iterates over the text in the response. - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of string. Each string chunk will be a text from the response :rtype: AsyncIterator[str] """ - async for byte in self.iter_bytes(chunk_size): # type: ignore + async for byte in self.iter_bytes(): # type: ignore text = byte.decode(self.encoding or "utf-8") yield text - async def iter_lines(self, chunk_size: int = None) -> AsyncIterator[str]: + async def iter_lines(self) -> AsyncIterator[str]: """Asynchronously iterates over the lines in the response. - :param int chunk_size: The maximum size of each chunk iterated over. :return: An async iterator of string. Each string chunk will be a line from the response :rtype: AsyncIterator[str] """ - async for text in self.iter_text(chunk_size): + async for text in self.iter_text(): lines = parse_lines_from_text(text) for line in lines: yield line diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py index 163135dad3af..6bf15a3f6c61 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py @@ -55,34 +55,6 @@ async def test_iter_with_error(client): raise ValueError("Should error before entering") assert response.is_closed -@pytest.mark.asyncio -async def test_iter_raw_with_chunksize_5(client): - request = HttpRequest("GET", "/streams/basic") - - async with client.send_request(request, stream=True) as response: - parts = [] - async for part in response.iter_raw(chunk_size=5): - parts.append(part) - assert parts == [b'Hello', b', wor', b'ld!'] - -@pytest.mark.asyncio -async def test_iter_raw_with_chunksize_13(client): - request = HttpRequest("GET", "/streams/basic") - async with client.send_request(request, stream=True) as response: - parts = [] - async for part in response.iter_raw(chunk_size=13): - parts.append(part) - assert parts == [b"Hello, world!"] - -@pytest.mark.asyncio -async def test_iter_raw_with_chunksize_20(client): - request = HttpRequest("GET", "/streams/basic") - async with client.send_request(request, stream=True) as response: - parts = [] - async for part in response.iter_raw(chunk_size=20): - parts.append(part) - assert parts == [b"Hello, world!"] - @pytest.mark.asyncio async def test_iter_raw_num_bytes_downloaded(client): request = HttpRequest("GET", "/streams/basic") @@ -107,34 +79,6 @@ async def test_iter_bytes(client): assert response.is_closed assert raw == b"Hello, world!" -@pytest.mark.asyncio -async def test_iter_bytes_with_chunk_size_5(client): - request = HttpRequest("GET", "/streams/basic") - - async with client.send_request(request, stream=True) as response: - parts = [] - async for part in response.iter_bytes(chunk_size=5): - parts.append(part) - assert parts == [b"Hello", b", wor", b"ld!"] - -@pytest.mark.asyncio -async def test_iter_bytes_with_chunk_size_13(client): - request = HttpRequest("GET", "/streams/basic") - async with client.send_request(request, stream=True) as response: - parts = [] - async for part in response.iter_bytes(chunk_size=13): - parts.append(part) - assert parts == [b"Hello, world!"] - -@pytest.mark.asyncio -async def test_iter_bytes_with_chunk_size_20(client): - request = HttpRequest("GET", "/streams/basic") - async with client.send_request(request, stream=True) as response: - parts = [] - async for part in response.iter_bytes(chunk_size=20): - parts.append(part) - assert parts == [b"Hello, world!"] - @pytest.mark.asyncio async def test_iter_text(client): request = HttpRequest("GET", "/basic/string") @@ -145,28 +89,6 @@ async def test_iter_text(client): content += part assert content == "Hello, world!" -@pytest.mark.asyncio -async def test_iter_text_with_chunk_size(client): - request = HttpRequest("GET", "/basic/string") - - async with client.send_request(request, stream=True) as response: - parts = [] - async for part in response.iter_text(chunk_size=5): - parts.append(part) - assert parts == ["Hello", ", wor", "ld!"] - - async with client.send_request(request, stream=True) as response: - parts = [] - async for part in response.iter_text(chunk_size=13): - parts.append(part) - assert parts == ["Hello, world!"] - - async with client.send_request(request, stream=True) as response: - parts = [] - async for part in response.iter_text(chunk_size=20): - parts.append(part) - assert parts == ["Hello, world!"] - @pytest.mark.asyncio async def test_iter_lines(client): request = HttpRequest("GET", "/basic/lines") diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py index 33d26cd654d7..88069c4be48b 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py @@ -64,21 +64,6 @@ def test_iter_with_error(client): raise ValueError("Should error before entering") assert response.is_closed -def test_iter_raw_with_chunksize(client): - request = HttpRequest("GET", "/streams/basic") - - with client.send_request(request, stream=True) as response: - parts = [part for part in response.iter_raw(chunk_size=5)] - assert parts == [b"Hello", b", wor", b"ld!"] - - with client.send_request(request, stream=True) as response: - parts = [part for part in response.iter_raw(chunk_size=13)] - assert parts == [b"Hello, world!"] - - with client.send_request(request, stream=True) as response: - parts = [part for part in response.iter_raw(chunk_size=20)] - assert parts == [b"Hello, world!"] - def test_iter_raw_num_bytes_downloaded(client): request = HttpRequest("GET", "/streams/basic") @@ -103,21 +88,6 @@ def test_iter_bytes(client): assert response.is_stream_consumed assert raw == b"Hello, world!" -def test_iter_bytes_with_chunk_size(client): - request = HttpRequest("GET", "/streams/basic") - - with client.send_request(request, stream=True) as response: - parts = [part for part in response.iter_bytes(chunk_size=5)] - assert parts == [b"Hello", b", wor", b"ld!"] - - with client.send_request(request, stream=True) as response: - parts = [part for part in response.iter_bytes(chunk_size=13)] - assert parts == [b"Hello, world!"] - - with client.send_request(request, stream=True) as response: - parts = [part for part in response.iter_bytes(chunk_size=20)] - assert parts == [b"Hello, world!"] - def test_iter_text(client): request = HttpRequest("GET", "/basic/string") @@ -127,21 +97,6 @@ def test_iter_text(client): content += part assert content == "Hello, world!" -def test_iter_text_with_chunk_size(client): - request = HttpRequest("GET", "/basic/string") - - with client.send_request(request, stream=True) as response: - parts = [part for part in response.iter_text(chunk_size=5)] - assert parts == ["Hello", ", wor", "ld!"] - - with client.send_request(request, stream=True) as response: - parts = [part for part in response.iter_text(chunk_size=13)] - assert parts == ["Hello, world!"] - - with client.send_request(request, stream=True) as response: - parts = [part for part in response.iter_text(chunk_size=20)] - assert parts == ["Hello, world!"] - def test_iter_lines(client): request = HttpRequest("GET", "/basic/lines") From c6860f29492ee91611d90eb1178c714607f1930d Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 12:57:26 -0400 Subject: [PATCH 50/64] move transport responses to rest module --- .../azure-core/azure/core/_pipeline_client.py | 7 +- .../azure/core/_pipeline_client_async.py | 10 +- .../azure-core/azure/core/pipeline/_tools.py | 51 ------- .../azure/core/pipeline/_tools_async.py | 46 ------ .../azure/core/pipeline/transport/_aiohttp.py | 81 +---------- .../pipeline/transport/_requests_asyncio.py | 36 +---- .../pipeline/transport/_requests_basic.py | 95 +------------ .../core/pipeline/transport/_requests_trio.py | 40 +----- .../azure-core/azure/core/rest/_aiohttp.py | 108 ++++++++++++++ .../azure-core/azure/core/rest/_helpers.py | 1 + .../azure/core/rest/_helpers_py3.py | 56 +++++++- .../azure/core/rest/_requests_asyncio.py | 55 ++++++++ .../azure/core/rest/_requests_basic.py | 132 ++++++++++++++++++ .../azure/core/rest/_requests_trio.py | 58 ++++++++ 14 files changed, 432 insertions(+), 344 deletions(-) create mode 100644 sdk/core/azure-core/azure/core/rest/_aiohttp.py create mode 100644 sdk/core/azure-core/azure/core/rest/_requests_asyncio.py create mode 100644 sdk/core/azure-core/azure/core/rest/_requests_basic.py create mode 100644 sdk/core/azure-core/azure/core/rest/_requests_trio.py diff --git a/sdk/core/azure-core/azure/core/_pipeline_client.py b/sdk/core/azure-core/azure/core/_pipeline_client.py index 6e6276b42c96..5e59b7a3d467 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client.py @@ -40,7 +40,6 @@ RetryPolicy, ) from .pipeline.transport import RequestsTransport -from .rest import HttpResponse try: from typing import TYPE_CHECKING @@ -59,8 +58,10 @@ Callable, Iterator, cast, + TypeVar ) # pylint: disable=unused-import - from .rest import HttpRequest + HTTPResponseType = TypeVar("HTTPResponseType") + HTTPRequestType = TypeVar("HTTPRequestType") _LOGGER = logging.getLogger(__name__) @@ -185,7 +186,7 @@ def _build_pipeline(self, config, **kwargs): # pylint: disable=no-self-use def send_request(self, request, **kwargs): - # type: (HttpRequest, Any) -> HttpResponse + # type: (HTTPRequestType, Any) -> HTTPResponseType """**Provisional** method that runs the network request through the client's chained policies. This method is marked as **provisional**, meaning it can be changed diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index 64b53d746b1b..0fcd8a37f9ee 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -40,11 +40,12 @@ from ._pipeline_client import _prepare_request try: - from typing import TYPE_CHECKING + from typing import TYPE_CHECKING, TypeVar except ImportError: TYPE_CHECKING = False -from .rest import HttpRequest, _AsyncContextManager, AsyncHttpResponse +HTTPRequestType = TypeVar("HTTPRequestType") +AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType") if TYPE_CHECKING: from typing import ( @@ -195,11 +196,11 @@ async def _make_pipeline_call(self, request, stream, **kwargs): def send_request( self, - request: HttpRequest, + request: HTTPRequestType, *, stream: bool = False, **kwargs: Any - ) -> Awaitable[AsyncHttpResponse]: + ) -> Awaitable[AsyncHTTPResponseType]: """**Provisional** method that runs the network request through the client's chained policies. This method is marked as **provisional**, meaning it can be changed. @@ -210,5 +211,6 @@ def send_request( :return: The response of your network call. Does not do error handling on your response. :rtype: ~azure.core.rest.AsyncHttpResponse """ + from .rest import _AsyncContextManager wrapped = self._make_pipeline_call(request, stream=stream, **kwargs) return _AsyncContextManager(wrapped=wrapped) diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools.py b/sdk/core/azure-core/azure/core/pipeline/_tools.py index eae6bcfa4437..4846bcb8d7c2 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools.py @@ -23,17 +23,6 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import TYPE_CHECKING -from ..exceptions import StreamClosedError, StreamConsumedError - - -if TYPE_CHECKING: - from typing import ( - Callable, - Optional, - Iterator, - ) - from azure.core.rest import HttpResponse def await_result(func, *args, **kwargs): """If func returns an awaitable, raise that this runner can't handle it.""" @@ -44,46 +33,6 @@ def await_result(func, *args, **kwargs): ) return result -def _stream_download_helper(decompress, stream_download_generator, response): - # type: (bool, Callable, HttpResponse) -> Iterator[bytes] - if response.is_stream_consumed: - raise StreamConsumedError() - if response.is_closed: - raise StreamClosedError() - - response.is_stream_consumed = True - stream_download = stream_download_generator( - pipeline=None, - response=response, - decompress=decompress, - ) - for part in stream_download: - response._num_bytes_downloaded += len(part) - yield part - -def iter_bytes_helper(stream_download_generator, response): - # type: (Callable, HttpResponse) -> Iterator[bytes] - if response._has_content(): # pylint: disable=protected-access - yield response._get_content() # pylint: disable=protected-access - else: - for part in _stream_download_helper( - decompress=True, - stream_download_generator=stream_download_generator, - response=response, - ): - yield part - response.close() - -def iter_raw_helper(stream_download_generator, response): - # type: (Callable, HttpResponse) -> Iterator[bytes] - for raw_bytes in _stream_download_helper( - decompress=False, - stream_download_generator=stream_download_generator, - response=response, - ): - yield raw_bytes - response.close() - def to_rest_response_helper(pipeline_transport_response, response_type): response = response_type( request=pipeline_transport_response.request._to_rest_request(), # pylint: disable=protected-access diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index a2b57546a624..d29988bd41ee 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -23,8 +23,6 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import Optional, Callable, AsyncIterator -from ..exceptions import StreamClosedError, StreamConsumedError async def await_result(func, *args, **kwargs): """If func returns an awaitable, await it.""" @@ -33,47 +31,3 @@ async def await_result(func, *args, **kwargs): # type ignore on await: https://github.com/python/mypy/issues/7587 return await result # type: ignore return result - -def _stream_download_helper( - decompress: bool, - stream_download_generator: Callable, - response, -) -> AsyncIterator[bytes]: - if response.is_stream_consumed: - raise StreamConsumedError() - if response.is_closed: - raise StreamClosedError() - - response.is_stream_consumed = True - return stream_download_generator( - pipeline=None, - response=response, - decompress=decompress, - ) - -async def iter_bytes_helper( - stream_download_generator: Callable, - response, -) -> AsyncIterator[bytes]: - if response._has_content(): # pylint: disable=protected-access - yield response._get_content() # pylint: disable=protected-access - else: - async for part in _stream_download_helper( - decompress=True, - stream_download_generator=stream_download_generator, - response=response, - ): - response._num_bytes_downloaded += len(part) - yield part - -async def iter_raw_helper( - stream_download_generator: Callable, - response, -) -> AsyncIterator[bytes]: - async for part in _stream_download_helper( - decompress=False, - stream_download_generator=stream_download_generator, - response=response, - ): - response._num_bytes_downloaded += len(part) - yield part diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 979e85638716..ab44728e4cd2 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -46,15 +46,7 @@ AsyncHttpTransport, AsyncHttpResponse, _ResponseStopIteration) -from ...rest import ( - HttpRequest as RestHttpRequest, - AsyncHttpResponse as RestAsyncHttpResponse, -) from .._tools import to_rest_response_helper, set_block_size -from .._tools_async import ( - iter_bytes_helper, - iter_raw_helper, -) # Matching requests, because why not? CONTENT_CHUNK_SIZE = 10 * 1024 @@ -369,76 +361,5 @@ def __getstate__(self): return state def _to_rest_response(self): + from ...rest._aiohttp import RestAioHttpTransportResponse return to_rest_response_helper(self, RestAioHttpTransportResponse) - -class RestAioHttpTransportResponse(RestAsyncHttpResponse): - def __init__( - self, - *, - request: RestHttpRequest, - internal_response, - ): - super().__init__(request=request, internal_response=internal_response) - self.status_code = internal_response.status - self.headers = CIMultiDict(internal_response.headers) # type: ignore - self.reason = internal_response.reason - self.content_type = internal_response.headers.get('content-type') - self._decompress = True - - @property - def text(self) -> str: - content = self.content - encoding = self.encoding - ctype = self.headers.get(aiohttp.hdrs.CONTENT_TYPE, "").lower() - mimetype = aiohttp.helpers.parse_mimetype(ctype) - - encoding = mimetype.parameters.get("charset") - if encoding: - try: - codecs.lookup(encoding) - except LookupError: - encoding = None - if not encoding: - if mimetype.type == "application" and ( - mimetype.subtype == "json" or mimetype.subtype == "rdap" - ): - # RFC 7159 states that the default encoding is UTF-8. - # RFC 7483 defines application/rdap+json - encoding = "utf-8" - elif content is None: - raise RuntimeError( - "Cannot guess the encoding of a not yet read content" - ) - else: - encoding = chardet.detect(content)["encoding"] - if not encoding: - encoding = "utf-8-sig" - - return content.decode(encoding) - - async def iter_raw(self) -> AsyncIteratorType[bytes]: - """Asynchronously iterates over the response's bytes. Will not decompress in the process - - :return: An async iterator of bytes from the response - :rtype: AsyncIterator[bytes] - """ - async for part in iter_raw_helper(AioHttpStreamDownloadGenerator, self): - yield part - await self.close() - - async def iter_bytes(self) -> AsyncIteratorType[bytes]: - """Asynchronously iterates over the response's bytes. Will decompress in the process - - :return: An async iterator of bytes from the response - :rtype: AsyncIterator[bytes] - """ - async for part in iter_bytes_helper(AioHttpStreamDownloadGenerator, self): - yield part - await self.close() - - def __getstate__(self): - state = self.__dict__.copy() - # Remove the unpicklable entries. - state['internal_response'] = None # aiohttp response are not pickable (see headers comments) - state['headers'] = CIMultiDict(self.headers) # MultiDictProxy is not pickable - return state diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index a05e39753b3d..8986fac5166c 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -42,16 +42,11 @@ AsyncHttpResponse, _ResponseStopIteration, _iterate_response_content) -from ._requests_basic import RequestsTransportResponse, _read_raw_stream, _RestRequestsTransportResponseBase +from ._requests_basic import RequestsTransportResponse, _read_raw_stream from ._base_requests_async import RequestsAsyncTransportBase from .._tools import to_rest_response_helper, set_block_size -from .._tools_async import ( - iter_bytes_helper, - iter_raw_helper -) -from ...rest import ( - AsyncHttpResponse as RestAsyncHttpResponse, -) + + _LOGGER = logging.getLogger(__name__) @@ -194,28 +189,5 @@ def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]: # typ return AsyncioStreamDownloadGenerator(pipeline, self, **kwargs) # type: ignore def _to_rest_response(self): + from ...rest._requests_asyncio import RestAsyncioRequestsTransportResponse return to_rest_response_helper(self, RestAsyncioRequestsTransportResponse) - -class RestAsyncioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore - """Asynchronous streaming of data from the response. - """ - - async def iter_raw(self) -> AsyncIteratorType[bytes]: - """Asynchronously iterates over the response's bytes. Will not decompress in the process - - :return: An async iterator of bytes from the response - :rtype: AsyncIterator[bytes] - """ - async for part in iter_raw_helper(AsyncioRequestsTransportResponse, self): - yield part - await self.close() - - async def iter_bytes(self) -> AsyncIteratorType[bytes]: - """Asynchronously iterates over the response's bytes. Will decompress in the process - - :return: An async iterator of bytes from the response - :rtype: AsyncIterator[bytes] - """ - async for part in iter_bytes_helper(AsyncioRequestsTransportResponse, self): - yield part - await self.close() diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py index 0d7f50f3fda1..4f98581a2ca9 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py @@ -36,27 +36,17 @@ from azure.core.configuration import ConnectionConfiguration from azure.core.exceptions import ( ServiceRequestError, - ServiceResponseError, - ResponseNotReadError, + ServiceResponseError ) from . import HttpRequest # pylint: disable=unused-import from ._base import ( HttpTransport, HttpResponse, - _HttpResponseBase, -) -from ...rest import ( - _HttpResponseBase as _RestHttpResponseBase, - HttpResponse as RestHttpResponse, + _HttpResponseBase ) from ._bigger_block_size_http_adapters import BiggerBlockSizeHTTPAdapter -from .._tools import ( - to_rest_response_helper, - iter_bytes_helper, - iter_raw_helper, - set_block_size, -) +from .._tools import to_rest_response_helper, set_block_size PipelineType = TypeVar("PipelineType") @@ -81,6 +71,7 @@ def _read_raw_stream(response, chunk_size=1): if not chunk: break yield chunk + # following behavior from requests iter_content, we set content consumed to True response._content_consumed = True # pylint: disable=protected-access @@ -173,58 +164,6 @@ def __next__(self): raise next = __next__ # Python 2 compatibility. -class _RestRequestsTransportResponseBase(_RestHttpResponseBase): - def __init__(self, **kwargs): - super(_RestRequestsTransportResponseBase, self).__init__(**kwargs) - self.status_code = self.internal_response.status_code - self.headers = self.internal_response.headers - self.reason = self.internal_response.reason - self.content_type = self.internal_response.headers.get('content-type') - - def _get_content(self): - """Return the internal response's content""" - if not self.internal_response._content_consumed: # pylint: disable=protected-access - # if we just call .content, requests will read in the content. - # we want to read it in our own way - return None - try: - return self.internal_response.content - except RuntimeError: - # requests throws a RuntimeError if the content for a response is already consumed - return None - - def _set_content(self, val): - """Set the internal response's content""" - self.internal_response._content = val # pylint: disable=protected-access - - def _has_content(self): - return self._get_content() is not None - - @_RestHttpResponseBase.encoding.setter # type: ignore - def encoding(self, value): - # type: (str) -> None - # ignoring setter bc of known mypy issue https://github.com/python/mypy/issues/1465 - self._encoding = value - encoding = value - if not encoding: - # There is a few situation where "requests" magic doesn't fit us: - # - https://github.com/psf/requests/issues/654 - # - https://github.com/psf/requests/issues/1737 - # - https://github.com/psf/requests/issues/2086 - from codecs import BOM_UTF8 - if self.internal_response.content[:3] == BOM_UTF8: - encoding = "utf-8-sig" - if encoding: - if encoding == "utf-8": - encoding = "utf-8-sig" - self.internal_response.encoding = encoding - - @property - def text(self): - if not self._has_content(): - raise ResponseNotReadError() - return self.internal_response.text - class RequestsTransportResponse(HttpResponse, _RequestsTransportResponseBase): """Streaming of data from the response. @@ -235,33 +174,9 @@ def stream_download(self, pipeline, **kwargs): return StreamDownloadGenerator(pipeline, self, **kwargs) def _to_rest_response(self): + from ...rest._requests_basic import RestRequestsTransportResponse return to_rest_response_helper(self, RestRequestsTransportResponse) -class RestRequestsTransportResponse(RestHttpResponse, _RestRequestsTransportResponseBase): - - def iter_bytes(self): - # type: () -> Iterator[bytes] - """Iterates over the response's bytes. Will decompress in the process - - :return: An iterator of bytes from the response - :rtype: Iterator[str] - """ - return iter_bytes_helper( - stream_download_generator=StreamDownloadGenerator, - response=self, - ) - - def iter_raw(self): - # type: () -> Iterator[bytes] - """Iterates over the response's bytes. Will not decompress in the process - - :return: An iterator of bytes from the response - :rtype: Iterator[str] - """ - return iter_raw_helper( - stream_download_generator=StreamDownloadGenerator, - response=self, - ) class RequestsTransport(HttpTransport): """Implements a basic requests HTTP sender. diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index 0a00a7f12988..37262860c4c4 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -42,16 +42,11 @@ AsyncHttpResponse, _ResponseStopIteration, _iterate_response_content) -from ._requests_basic import RequestsTransportResponse, _read_raw_stream, _RestRequestsTransportResponseBase +from ._requests_basic import RequestsTransportResponse, _read_raw_stream from ._base_requests_async import RequestsAsyncTransportBase from .._tools import to_rest_response_helper, set_block_size -from .._tools_async import ( - iter_raw_helper, - iter_bytes_helper, -) -from ...rest import ( - AsyncHttpResponse as RestAsyncHttpResponse, -) + + _LOGGER = logging.getLogger(__name__) @@ -114,36 +109,9 @@ def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]: # ty return TrioStreamDownloadGenerator(pipeline, self, **kwargs) def _to_rest_response(self): + from ...rest._requests_trio import RestTrioRequestsTransportResponse return to_rest_response_helper(self, RestTrioRequestsTransportResponse) -class RestTrioRequestsTransportResponse(RestAsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore - """Asynchronous streaming of data from the response. - """ - async def iter_raw(self) -> AsyncIteratorType[bytes]: - """Asynchronously iterates over the response's bytes. Will not decompress in the process - - :return: An async iterator of bytes from the response - :rtype: AsyncIterator[bytes] - """ - async for part in iter_raw_helper(TrioStreamDownloadGenerator, self): - yield part - await self.close() - - async def iter_bytes(self) -> AsyncIteratorType[bytes]: - """Asynchronously iterates over the response's bytes. Will decompress in the process - - :return: An async iterator of bytes from the response - :rtype: AsyncIterator[bytes] - """ - async for part in iter_bytes_helper(TrioStreamDownloadGenerator, self): - yield part - await self.close() - - async def close(self) -> None: - self.is_closed = True - self.internal_response.close() - await trio.sleep(0) - class TrioRequestsTransport(RequestsAsyncTransportBase): # type: ignore """Identical implementation as the synchronous RequestsTransport wrapped in a class with diff --git a/sdk/core/azure-core/azure/core/rest/_aiohttp.py b/sdk/core/azure-core/azure/core/rest/_aiohttp.py new file mode 100644 index 000000000000..61e35003b1e4 --- /dev/null +++ b/sdk/core/azure-core/azure/core/rest/_aiohttp.py @@ -0,0 +1,108 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +import codecs +from typing import AsyncIterator +from multidict import CIMultiDict +import aiohttp +from . import HttpRequest, AsyncHttpResponse +try: + import cchardet as chardet +except ImportError: # pragma: no cover + import chardet # type: ignore +from ._helpers_py3 import iter_raw_helper, iter_bytes_helper +from ..pipeline.transport._aiohttp import AioHttpStreamDownloadGenerator + + +class RestAioHttpTransportResponse(AsyncHttpResponse): + def __init__( + self, + *, + request: HttpRequest, + internal_response, + ): + super().__init__(request=request, internal_response=internal_response) + self.status_code = internal_response.status + self.headers = CIMultiDict(internal_response.headers) # type: ignore + self.reason = internal_response.reason + self.content_type = internal_response.headers.get('content-type') + self._decompress = True + + @property + def text(self) -> str: + content = self.content + encoding = self.encoding + ctype = self.headers.get(aiohttp.hdrs.CONTENT_TYPE, "").lower() + mimetype = aiohttp.helpers.parse_mimetype(ctype) + + encoding = mimetype.parameters.get("charset") + if encoding: + try: + codecs.lookup(encoding) + except LookupError: + encoding = None + if not encoding: + if mimetype.type == "application" and ( + mimetype.subtype == "json" or mimetype.subtype == "rdap" + ): + # RFC 7159 states that the default encoding is UTF-8. + # RFC 7483 defines application/rdap+json + encoding = "utf-8" + elif content is None: + raise RuntimeError( + "Cannot guess the encoding of a not yet read content" + ) + else: + encoding = chardet.detect(content)["encoding"] + if not encoding: + encoding = "utf-8-sig" + + return content.decode(encoding) + + async def iter_raw(self) -> AsyncIterator[bytes]: + """Asynchronously iterates over the response's bytes. Will not decompress in the process + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_raw_helper(AioHttpStreamDownloadGenerator, self): + yield part + await self.close() + + async def iter_bytes(self) -> AsyncIterator[bytes]: + """Asynchronously iterates over the response's bytes. Will decompress in the process + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_bytes_helper(AioHttpStreamDownloadGenerator, self): + yield part + await self.close() + + def __getstate__(self): + state = self.__dict__.copy() + # Remove the unpicklable entries. + state['internal_response'] = None # aiohttp response are not pickable (see headers comments) + state['headers'] = CIMultiDict(self.headers) # MultiDictProxy is not pickable + return state diff --git a/sdk/core/azure-core/azure/core/rest/_helpers.py b/sdk/core/azure-core/azure/core/rest/_helpers.py index fa309dabc097..5caacb500265 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers.py @@ -41,6 +41,7 @@ Iterable, Iterator, cast, + Callable, ) import xml.etree.ElementTree as ET import six diff --git a/sdk/core/azure-core/azure/core/rest/_helpers_py3.py b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py index 310e5467d959..d7e09f87ecb6 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py @@ -24,9 +24,17 @@ # # -------------------------------------------------------------------------- import collections.abc -from typing import AsyncIterable, Dict, Iterable, Tuple, Union +from typing import ( + AsyncIterable, + Dict, + Iterable, + Tuple, + Union, + Callable, + AsyncIterator as AsyncIteratorType +) +from ..exceptions import StreamConsumedError, StreamClosedError -from six import Iterator from ._helpers import ( _shared_set_content_body, HeadersType @@ -45,3 +53,47 @@ def set_content_body(content: ContentType) -> Tuple[ "Unexpected type for 'content': '{}'. ".format(type(content)) + "We expect 'content' to either be str, bytes, or an Iterable / AsyncIterable" ) + +def _stream_download_helper( + decompress: bool, + stream_download_generator: Callable, + response, +) -> AsyncIteratorType[bytes]: + if response.is_stream_consumed: + raise StreamConsumedError() + if response.is_closed: + raise StreamClosedError() + + response.is_stream_consumed = True + return stream_download_generator( + pipeline=None, + response=response, + decompress=decompress, + ) + +async def iter_bytes_helper( + stream_download_generator: Callable, + response, +) -> AsyncIteratorType[bytes]: + if response._has_content(): # pylint: disable=protected-access + yield response._get_content() # pylint: disable=protected-access + else: + async for part in _stream_download_helper( + decompress=True, + stream_download_generator=stream_download_generator, + response=response, + ): + response._num_bytes_downloaded += len(part) + yield part + +async def iter_raw_helper( + stream_download_generator: Callable, + response, +) -> AsyncIteratorType[bytes]: + async for part in _stream_download_helper( + decompress=False, + stream_download_generator=stream_download_generator, + response=response, + ): + response._num_bytes_downloaded += len(part) + yield part diff --git a/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py b/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py new file mode 100644 index 000000000000..f17fc680d14e --- /dev/null +++ b/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py @@ -0,0 +1,55 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from typing import AsyncIterator +from ._helpers_py3 import iter_bytes_helper, iter_raw_helper +from . import AsyncHttpResponse +from ._requests_basic import _RestRequestsTransportResponseBase +from ..pipeline.transport._requests_asyncio import AsyncioStreamDownloadGenerator + +class RestAsyncioRequestsTransportResponse(AsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore + """Asynchronous streaming of data from the response. + """ + + async def iter_raw(self) -> AsyncIterator[bytes]: + """Asynchronously iterates over the response's bytes. Will not decompress in the process + + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + + async for part in iter_raw_helper(AsyncioStreamDownloadGenerator, self): + yield part + await self.close() + + async def iter_bytes(self) -> AsyncIterator[bytes]: + """Asynchronously iterates over the response's bytes. Will decompress in the process + + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_bytes_helper(AsyncioStreamDownloadGenerator, self): + yield part + await self.close() diff --git a/sdk/core/azure-core/azure/core/rest/_requests_basic.py b/sdk/core/azure-core/azure/core/rest/_requests_basic.py new file mode 100644 index 000000000000..4679d9e20f1e --- /dev/null +++ b/sdk/core/azure-core/azure/core/rest/_requests_basic.py @@ -0,0 +1,132 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from typing import TYPE_CHECKING + +from ..exceptions import ResponseNotReadError, StreamConsumedError, StreamClosedError +from . import _HttpResponseBase, HttpResponse +from ..pipeline.transport._requests_basic import StreamDownloadGenerator + +if TYPE_CHECKING: + from typing import Iterator + +class _RestRequestsTransportResponseBase(_HttpResponseBase): + def __init__(self, **kwargs): + super(_RestRequestsTransportResponseBase, self).__init__(**kwargs) + self.status_code = self.internal_response.status_code + self.headers = self.internal_response.headers + self.reason = self.internal_response.reason + self.content_type = self.internal_response.headers.get('content-type') + + def _get_content(self): + """Return the internal response's content""" + if not self.internal_response._content_consumed: # pylint: disable=protected-access + # if we just call .content, requests will read in the content. + # we want to read it in our own way + return None + try: + return self.internal_response.content + except RuntimeError: + # requests throws a RuntimeError if the content for a response is already consumed + return None + + def _set_content(self, val): + """Set the internal response's content""" + self.internal_response._content = val # pylint: disable=protected-access + + def _has_content(self): + return self._get_content() is not None + + @_HttpResponseBase.encoding.setter # type: ignore + def encoding(self, value): + # type: (str) -> None + # ignoring setter bc of known mypy issue https://github.com/python/mypy/issues/1465 + self._encoding = value + encoding = value + if not encoding: + # There is a few situation where "requests" magic doesn't fit us: + # - https://github.com/psf/requests/issues/654 + # - https://github.com/psf/requests/issues/1737 + # - https://github.com/psf/requests/issues/2086 + from codecs import BOM_UTF8 + if self.internal_response.content[:3] == BOM_UTF8: + encoding = "utf-8-sig" + if encoding: + if encoding == "utf-8": + encoding = "utf-8-sig" + self.internal_response.encoding = encoding + + @property + def text(self): + if not self._has_content(): + raise ResponseNotReadError() + return self.internal_response.text + +def _stream_download_helper(decompress, response): + if response.is_stream_consumed: + raise StreamConsumedError() + if response.is_closed: + raise StreamClosedError() + + response.is_stream_consumed = True + stream_download = StreamDownloadGenerator( + pipeline=None, + response=response, + decompress=decompress, + ) + for part in stream_download: + response._num_bytes_downloaded += len(part) + yield part + +class RestRequestsTransportResponse(HttpResponse, _RestRequestsTransportResponseBase): + + def iter_bytes(self): + # type: () -> Iterator[bytes] + """Iterates over the response's bytes. Will decompress in the process + :return: An iterator of bytes from the response + :rtype: Iterator[str] + """ + if self._has_content(): + yield self._get_content() + else: + for part in _stream_download_helper( + decompress=True, + response=self, + ): + yield part + self.close() + + def iter_raw(self): + # type: () -> Iterator[bytes] + """Iterates over the response's bytes. Will not decompress in the process + :return: An iterator of bytes from the response + :rtype: Iterator[str] + """ + for raw_bytes in _stream_download_helper( + decompress=False, + response=self, + ): + yield raw_bytes + self.close() diff --git a/sdk/core/azure-core/azure/core/rest/_requests_trio.py b/sdk/core/azure-core/azure/core/rest/_requests_trio.py new file mode 100644 index 000000000000..4dd81247e6ff --- /dev/null +++ b/sdk/core/azure-core/azure/core/rest/_requests_trio.py @@ -0,0 +1,58 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from typing import AsyncIterator +import trio +from . import AsyncHttpResponse +from ._requests_basic import _RestRequestsTransportResponseBase +from ._helpers_py3 import iter_bytes_helper, iter_raw_helper +from ..pipeline.transport._requests_trio import TrioStreamDownloadGenerator + +class RestTrioRequestsTransportResponse(AsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore + """Asynchronous streaming of data from the response. + """ + async def iter_raw(self) -> AsyncIterator[bytes]: + """Asynchronously iterates over the response's bytes. Will not decompress in the process + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + async for part in iter_raw_helper(TrioStreamDownloadGenerator, self): + yield part + await self.close() + + async def iter_bytes(self) -> AsyncIterator[bytes]: + """Asynchronously iterates over the response's bytes. Will decompress in the process + :return: An async iterator of bytes from the response + :rtype: AsyncIterator[bytes] + """ + + async for part in iter_bytes_helper(TrioStreamDownloadGenerator, self): + yield part + await self.close() + + async def close(self) -> None: + self.is_closed = True + self.internal_response.close() + await trio.sleep(0) From 8b3c812d91d2c8c02860fe7ba2c96c7c5172d1fe Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 13:30:12 -0400 Subject: [PATCH 51/64] move asyncio waiting for internal response's close onto transport responses --- sdk/core/azure-core/azure/core/rest/_aiohttp.py | 11 +++++++++++ .../azure-core/azure/core/rest/_requests_asyncio.py | 11 +++++++++++ sdk/core/azure-core/azure/core/rest/_rest_py3.py | 4 +--- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/rest/_aiohttp.py b/sdk/core/azure-core/azure/core/rest/_aiohttp.py index 61e35003b1e4..98cf2d1706b6 100644 --- a/sdk/core/azure-core/azure/core/rest/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/rest/_aiohttp.py @@ -24,6 +24,7 @@ # # -------------------------------------------------------------------------- +import asyncio import codecs from typing import AsyncIterator from multidict import CIMultiDict @@ -106,3 +107,13 @@ def __getstate__(self): state['internal_response'] = None # aiohttp response are not pickable (see headers comments) state['headers'] = CIMultiDict(self.headers) # MultiDictProxy is not pickable return state + + async def close(self) -> None: + """Close the response. + + :return: None + :rtype: None + """ + self.is_closed = True + self.internal_response.close() + await asyncio.sleep(0) diff --git a/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py b/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py index f17fc680d14e..59797e838937 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py @@ -24,6 +24,7 @@ # # -------------------------------------------------------------------------- from typing import AsyncIterator +import asyncio from ._helpers_py3 import iter_bytes_helper, iter_raw_helper from . import AsyncHttpResponse from ._requests_basic import _RestRequestsTransportResponseBase @@ -53,3 +54,13 @@ async def iter_bytes(self) -> AsyncIterator[bytes]: async for part in iter_bytes_helper(AsyncioStreamDownloadGenerator, self): yield part await self.close() + + async def close(self) -> None: + """Close the response. + + :return: None + :rtype: None + """ + self.is_closed = True + self.internal_response.close() + await asyncio.sleep(0) diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index 3c2296e5978e..f21706ae3a3d 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -24,7 +24,6 @@ # # -------------------------------------------------------------------------- import copy -import asyncio import cgi import collections import collections.abc @@ -507,8 +506,7 @@ async def close(self) -> None: :rtype: None """ self.is_closed = True - self.internal_response.close() - await asyncio.sleep(0) + await self.internal_response.close() async def __aexit__(self, *args) -> None: await self.close() From f9ecfdbed027473739ff34c51e2a965ce86b8f86 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 13:32:54 -0400 Subject: [PATCH 52/64] fix wording of StreamClosedError --- sdk/core/azure-core/azure/core/exceptions.py | 4 ++-- .../async_tests/test_rest_stream_responses_async.py | 2 +- .../tests/testserver_tests/test_rest_stream_responses.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/exceptions.py b/sdk/core/azure-core/azure/core/exceptions.py index a2282657b17a..e58b40274901 100644 --- a/sdk/core/azure-core/azure/core/exceptions.py +++ b/sdk/core/azure-core/azure/core/exceptions.py @@ -457,8 +457,8 @@ class StreamClosedError(Exception): """ def __init__(self): message = ( - "You can not try to read or stream this response's content, since the " - "response's stream has been closed." + "The response's content can no longer be read or streamed, since the " + "response has already been closed." ) super(StreamClosedError, self).__init__(message) diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py index 6bf15a3f6c61..2e290d84bea6 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py @@ -134,7 +134,7 @@ async def test_cannot_read_after_response_closed(client): with pytest.raises(StreamClosedError) as ex: await response.read() - assert "You can not try to read or stream this response's content, since the response's stream has been closed" in str(ex.value) + assert "The response's content can no longer be read or streamed, since the response has already been closed." in str(ex.value) @pytest.mark.asyncio async def test_decompress_plain_no_header(client): diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py index 88069c4be48b..0854bda54bd5 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py @@ -141,7 +141,7 @@ def test_cannot_read_after_response_closed(client): response.close() with pytest.raises(StreamClosedError) as ex: response.read() - assert "You can not try to read or stream this response's content, since the response's stream has been closed" in str(ex.value) + assert "The response's content can no longer be read or streamed, since the response has already been closed." in str(ex.value) def test_decompress_plain_no_header(client): # thanks to Xiang Yan for this test! From 16ed67b2dd4f6ea1e4873064ffacb4c850ce93ad Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 13:44:16 -0400 Subject: [PATCH 53/64] fix return_pipeline_response's http_request property --- .../azure-core/azure/core/_pipeline_client.py | 1 + .../azure/core/_pipeline_client_async.py | 1 + .../test_rest_http_response_async.py | 11 ++++++++ .../test_rest_stream_responses_async.py | 16 ++++++++++- .../test_rest_http_response.py | 27 +++++++------------ .../test_rest_stream_responses.py | 10 ++++++- 6 files changed, 46 insertions(+), 20 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client.py b/sdk/core/azure-core/azure/core/_pipeline_client.py index 5e59b7a3d467..cfb50d58087b 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client.py @@ -208,5 +208,6 @@ def send_request(self, request, **kwargs): response.close() if return_pipeline_response: pipeline_response.http_response = response + pipeline_response.http_request = request return pipeline_response return response diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index 0fcd8a37f9ee..9bfb96fbe654 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -191,6 +191,7 @@ async def _make_pipeline_call(self, request, stream, **kwargs): response = rest_response if return_pipeline_response: pipeline_response.http_response = response + pipeline_response.http_request = request return pipeline_response return response diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py index e763c886bf81..61066ee4a35e 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py @@ -241,6 +241,17 @@ async def test_multipart_files_content(send_request): ) await send_request(request) +@pytest.mark.asyncio +async def test_send_request_return_pipeline_response(client): + # we use return_pipeline_response for some cases in autorest + request = HttpRequest("GET", "/basic/string") + response = await client.send_request(request, _return_pipeline_response=True) + assert hasattr(response, "http_request") + assert hasattr(response, "http_response") + assert hasattr(response, "context") + assert response.http_response.text == "Hello, world!" + assert hasattr(response.http_request, "content") + # @pytest.mark.asyncio # async def test_multipart_encode_non_seekable_filelike(send_request): # """ diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py index 2e290d84bea6..5732c5f1f48e 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py @@ -181,4 +181,18 @@ async def test_iter_read_back_and_forth(client): with pytest.raises(StreamConsumedError): await response.read() with pytest.raises(ResponseNotReadError): - response.text \ No newline at end of file + response.text + +@pytest.mark.asyncio +async def test_stream_with_return_pipeline_response(client): + request = HttpRequest("GET", "/basic/lines") + pipeline_response = await client.send_request(request, stream=True, _return_pipeline_response=True) + assert hasattr(pipeline_response, "http_request") + assert hasattr(pipeline_response.http_request, "content") + assert hasattr(pipeline_response, "http_response") + assert hasattr(pipeline_response, "context") + parts = [] + async for line in pipeline_response.http_response.iter_lines(): + parts.append(line) + assert parts == ['Hello,\n', 'world!'] + await client.close() diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py index d31d54522a78..804a98b8890e 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py @@ -287,21 +287,12 @@ def test_put_xml_basic(send_request): ) send_request(request) -class MockHttpRequest(HttpRequest): - """Use this to check how many times _convert() was called""" - def __init__(self, *args, **kwargs): - super(MockHttpRequest, self).__init__(*args, **kwargs) - self.num_calls_to_convert = 0 - - def _convert(self): - self.num_calls_to_convert += 1 - return super(MockHttpRequest, self)._convert() - - -def test_request_no_conversion(send_request): - request = MockHttpRequest("GET", "/basic/string") - response = send_request( - request=request, - ) - assert response.status_code == 200 - assert request.num_calls_to_convert == 0 \ No newline at end of file +def test_send_request_return_pipeline_response(client): + # we use return_pipeline_response for some cases in autorest + request = HttpRequest("GET", "/basic/string") + response = client.send_request(request, _return_pipeline_response=True) + assert hasattr(response, "http_request") + assert hasattr(response, "http_response") + assert hasattr(response, "context") + assert response.http_response.text == "Hello, world!" + assert hasattr(response.http_request, "content") diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py index 0854bda54bd5..004f366c1659 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py @@ -212,4 +212,12 @@ def test_iter_read_back_and_forth(client): with pytest.raises(StreamConsumedError): response.read() with pytest.raises(ResponseNotReadError): - response.text \ No newline at end of file + response.text + +def test_stream_with_return_pipeline_response(client): + request = HttpRequest("GET", "/basic/lines") + pipeline_response = client.send_request(request, stream=True, _return_pipeline_response=True) + assert hasattr(pipeline_response, "http_request") + assert hasattr(pipeline_response, "http_response") + assert hasattr(pipeline_response, "context") + assert list(pipeline_response.http_response.iter_lines()) == ['Hello,\n', 'world!'] From 934f53da21b62ec7eb5825d8f40059621d05a710 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 14:11:11 -0400 Subject: [PATCH 54/64] inclide zlib compressing of aiohttp body --- .../azure/core/pipeline/transport/_aiohttp.py | 4 +++- .../azure-core/azure/core/rest/_aiohttp.py | 20 ++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index ab44728e4cd2..36e5dc16888c 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -362,4 +362,6 @@ def __getstate__(self): def _to_rest_response(self): from ...rest._aiohttp import RestAioHttpTransportResponse - return to_rest_response_helper(self, RestAioHttpTransportResponse) + response = to_rest_response_helper(self, RestAioHttpTransportResponse) + response._decompress = self._decompress # pylint: disable=protected-access + return response diff --git a/sdk/core/azure-core/azure/core/rest/_aiohttp.py b/sdk/core/azure-core/azure/core/rest/_aiohttp.py index 98cf2d1706b6..320462ba4091 100644 --- a/sdk/core/azure-core/azure/core/rest/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/rest/_aiohttp.py @@ -50,7 +50,25 @@ def __init__( self.headers = CIMultiDict(internal_response.headers) # type: ignore self.reason = internal_response.reason self.content_type = internal_response.headers.get('content-type') - self._decompress = True + self._decompress = None + self._decompressed_content = None + + def _get_content(self): + if not self._decompress: + return self._content + enc = self.headers.get('Content-Encoding') + if not enc: + return self._content + enc = enc.lower() + if enc in ("gzip", "deflate"): + if self._decompressed_content: + return self._decompressed_content + import zlib + zlib_mode = 16 + zlib.MAX_WBITS if enc == "gzip" else zlib.MAX_WBITS + decompressor = zlib.decompressobj(wbits=zlib_mode) + self._decompressed_content = decompressor.decompress(self._content) + return self._decompressed_content + return self._content @property def text(self) -> str: From fcbd91d812db631b1d119499354e9f12aaf2c016 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 14:22:36 -0400 Subject: [PATCH 55/64] add error reading tests --- .../test_rest_stream_responses_async.py | 15 +++++++++++++++ .../test_rest_stream_responses.py | 13 +++++++++++++ 2 files changed, 28 insertions(+) diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py index 5732c5f1f48e..c41f7f290818 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py @@ -196,3 +196,18 @@ async def test_stream_with_return_pipeline_response(client): parts.append(line) assert parts == ['Hello,\n', 'world!'] await client.close() + +@pytest.mark.asyncio +async def test_error_reading(client): + request = HttpRequest("GET", "/errors/403") + async with client.send_request(request, stream=True) as response: + await response.read() + assert response.content == b"" + response.content + + response = await client.send_request(request, stream=True) + with pytest.raises(HttpResponseError): + response.raise_for_status() + await response.read() + assert response.content == b"" + await client.close() diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py index 004f366c1659..3b0e6469e948 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py @@ -221,3 +221,16 @@ def test_stream_with_return_pipeline_response(client): assert hasattr(pipeline_response, "http_response") assert hasattr(pipeline_response, "context") assert list(pipeline_response.http_response.iter_lines()) == ['Hello,\n', 'world!'] + +def test_error_reading(client): + request = HttpRequest("GET", "/errors/403") + with client.send_request(request, stream=True) as response: + response.read() + assert response.content == b"" + + response = client.send_request(request, stream=True) + with pytest.raises(HttpResponseError): + response.raise_for_status() + response.read() + assert response.content == b"" + response.content From 26672394c49508557cbea6b12099fb0a3ee93fe5 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 14:53:01 -0400 Subject: [PATCH 56/64] clear up content setting --- .../azure-core/azure/core/rest/_aiohttp.py | 12 +++++- .../azure/core/rest/_helpers_py3.py | 6 ++- .../azure/core/rest/_requests_asyncio.py | 21 ++++++++- .../azure/core/rest/_requests_basic.py | 43 ++++++++++++++++--- .../azure/core/rest/_requests_trio.py | 21 ++++++++- sdk/core/azure-core/azure/core/rest/_rest.py | 26 +++-------- .../azure-core/azure/core/rest/_rest_py3.py | 32 +++++--------- 7 files changed, 105 insertions(+), 56 deletions(-) diff --git a/sdk/core/azure-core/azure/core/rest/_aiohttp.py b/sdk/core/azure-core/azure/core/rest/_aiohttp.py index 320462ba4091..cd09a8688cc5 100644 --- a/sdk/core/azure-core/azure/core/rest/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/rest/_aiohttp.py @@ -36,6 +36,7 @@ import chardet # type: ignore from ._helpers_py3 import iter_raw_helper, iter_bytes_helper from ..pipeline.transport._aiohttp import AioHttpStreamDownloadGenerator +from ..exceptions import ResponseNotReadError class RestAioHttpTransportResponse(AsyncHttpResponse): @@ -53,7 +54,10 @@ def __init__( self._decompress = None self._decompressed_content = None - def _get_content(self): + @property + def content(self) -> bytes: + if self._content is None: + raise ResponseNotReadError() if not self._decompress: return self._content enc = self.headers.get('Content-Encoding') @@ -115,7 +119,11 @@ async def iter_bytes(self) -> AsyncIterator[bytes]: :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_bytes_helper(AioHttpStreamDownloadGenerator, self): + async for part in iter_bytes_helper( + AioHttpStreamDownloadGenerator, + self, + content=self._content + ): yield part await self.close() diff --git a/sdk/core/azure-core/azure/core/rest/_helpers_py3.py b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py index d7e09f87ecb6..5c4efd78f213 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py @@ -31,6 +31,7 @@ Tuple, Union, Callable, + Optional, AsyncIterator as AsyncIteratorType ) from ..exceptions import StreamConsumedError, StreamClosedError @@ -74,9 +75,10 @@ def _stream_download_helper( async def iter_bytes_helper( stream_download_generator: Callable, response, + content: Optional[bytes], ) -> AsyncIteratorType[bytes]: - if response._has_content(): # pylint: disable=protected-access - yield response._get_content() # pylint: disable=protected-access + if content: # pylint: disable=protected-access + yield content # pylint: disable=protected-access else: async for part in _stream_download_helper( decompress=True, diff --git a/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py b/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py index 59797e838937..98986ec2b07f 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py @@ -27,7 +27,7 @@ import asyncio from ._helpers_py3 import iter_bytes_helper, iter_raw_helper from . import AsyncHttpResponse -from ._requests_basic import _RestRequestsTransportResponseBase +from ._requests_basic import _RestRequestsTransportResponseBase, _has_content from ..pipeline.transport._requests_asyncio import AsyncioStreamDownloadGenerator class RestAsyncioRequestsTransportResponse(AsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore @@ -51,7 +51,11 @@ async def iter_bytes(self) -> AsyncIterator[bytes]: :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - async for part in iter_bytes_helper(AsyncioStreamDownloadGenerator, self): + async for part in iter_bytes_helper( + AsyncioStreamDownloadGenerator, + self, + content=self.content if _has_content(self) else None + ): yield part await self.close() @@ -64,3 +68,16 @@ async def close(self) -> None: self.is_closed = True self.internal_response.close() await asyncio.sleep(0) + + async def read(self) -> bytes: + """Read the response's bytes into memory. + + :return: The response's bytes + :rtype: bytes + """ + if not _has_content(self): + parts = [] + async for part in self.iter_bytes(): # type: ignore + parts.append(part) + self.internal_response._content = b"".join(parts) # pylint: disable=protected-access + return self.content diff --git a/sdk/core/azure-core/azure/core/rest/_requests_basic.py b/sdk/core/azure-core/azure/core/rest/_requests_basic.py index 4679d9e20f1e..935cc545f78c 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_basic.py @@ -32,6 +32,13 @@ if TYPE_CHECKING: from typing import Iterator +def _has_content(response): + try: + response.content # pylint: disable=pointless-statement + return True + except ResponseNotReadError: + return False + class _RestRequestsTransportResponseBase(_HttpResponseBase): def __init__(self, **kwargs): super(_RestRequestsTransportResponseBase, self).__init__(**kwargs) @@ -40,6 +47,20 @@ def __init__(self, **kwargs): self.reason = self.internal_response.reason self.content_type = self.internal_response.headers.get('content-type') + @property + def content(self): + # type: () -> bytes + if not self.internal_response._content_consumed: # pylint: disable=protected-access + # if we just call .content, requests will read in the content. + # we want to read it in our own way + raise ResponseNotReadError() + + try: + return self.internal_response.content + except RuntimeError: + # requests throws a RuntimeError if the content for a response is already consumed + raise ResponseNotReadError() + def _get_content(self): """Return the internal response's content""" if not self.internal_response._content_consumed: # pylint: disable=protected-access @@ -56,9 +77,6 @@ def _set_content(self, val): """Set the internal response's content""" self.internal_response._content = val # pylint: disable=protected-access - def _has_content(self): - return self._get_content() is not None - @_HttpResponseBase.encoding.setter # type: ignore def encoding(self, value): # type: (str) -> None @@ -80,8 +98,8 @@ def encoding(self, value): @property def text(self): - if not self._has_content(): - raise ResponseNotReadError() + # this will trigger errors if response is not read in + self.content # pylint: disable=pointless-statement return self.internal_response.text def _stream_download_helper(decompress, response): @@ -108,8 +126,8 @@ def iter_bytes(self): :return: An iterator of bytes from the response :rtype: Iterator[str] """ - if self._has_content(): - yield self._get_content() + if _has_content(self): + yield self.content else: for part in _stream_download_helper( decompress=True, @@ -130,3 +148,14 @@ def iter_raw(self): ): yield raw_bytes self.close() + + def read(self): + # type: () -> bytes + """Read the response's bytes. + + :return: The read in bytes + :rtype: bytes + """ + if not _has_content(self): + self.internal_response._content = b"".join(self.iter_bytes()) # pylint: disable=protected-access + return self.content diff --git a/sdk/core/azure-core/azure/core/rest/_requests_trio.py b/sdk/core/azure-core/azure/core/rest/_requests_trio.py index 4dd81247e6ff..9834bff51def 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_trio.py @@ -26,7 +26,7 @@ from typing import AsyncIterator import trio from . import AsyncHttpResponse -from ._requests_basic import _RestRequestsTransportResponseBase +from ._requests_basic import _RestRequestsTransportResponseBase, _has_content from ._helpers_py3 import iter_bytes_helper, iter_raw_helper from ..pipeline.transport._requests_trio import TrioStreamDownloadGenerator @@ -48,10 +48,27 @@ async def iter_bytes(self) -> AsyncIterator[bytes]: :rtype: AsyncIterator[bytes] """ - async for part in iter_bytes_helper(TrioStreamDownloadGenerator, self): + async for part in iter_bytes_helper( + TrioStreamDownloadGenerator, + self, + content=self.content if _has_content(self) else None + ): yield part await self.close() + async def read(self) -> bytes: + """Read the response's bytes into memory. + + :return: The response's bytes + :rtype: bytes + """ + if not _has_content(self): + parts = [] + async for part in self.iter_bytes(): # type: ignore + parts.append(part) + self.internal_response._content = b"".join(parts) # pylint: disable=protected-access + return self.content + async def close(self) -> None: self.is_closed = True self.internal_response.close() diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index fc434014ffbc..5d7f366a2de2 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -198,7 +198,7 @@ def __init__(self, **kwargs): self.content_type = None self._json = None # this is filled in ContentDecodePolicy, when we deserialize self._connection_data_block_size = None - self._content = None + self._content = None # type: Optional[bytes] @property def url(self): @@ -217,18 +217,6 @@ def _get_charset_encoding(self): return None return encoding - def _get_content(self): - """Return the internal response's content""" - return self._content - - def _set_content(self, val): - """Set the internal response's content""" - self._content = val - - def _has_content(self): - """How to check if your internal response has content""" - return self._content is not None - @property def encoding(self): # type: (...) -> Optional[str] @@ -270,8 +258,8 @@ def json(self): :rtype: any :raises json.decoder.JSONDecodeError or ValueError (in python 2.7) if object is not JSON decodable: """ - if not self._has_content(): - raise ResponseNotReadError() + # this will trigger errors if response is not read in + self.content # pylint: disable=pointless-statement if not self._json: self._json = loads(self.text) return self._json @@ -289,9 +277,9 @@ def raise_for_status(self): def content(self): # type: (...) -> bytes """Return the response's content in bytes.""" - if not self._has_content(): + if self._content is None: raise ResponseNotReadError() - return cast(bytes, self._get_content()) + return self._content def __repr__(self): # type: (...) -> str @@ -348,8 +336,8 @@ def read(self): Read the response's bytes. """ - if not self._has_content(): - self._set_content(b"".join(self.iter_bytes())) + if self._content is None: + self._content = b"".join(self.iter_bytes()) return self.content def iter_raw(self): diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index f21706ae3a3d..bac206a80c53 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -227,7 +227,7 @@ def __init__( self.content_type = None self._connection_data_block_size = None self._json = None # this is filled in ContentDecodePolicy, when we deserialize - self._content = None + self._content = None # type: Optional[bytes] @property def url(self) -> str: @@ -245,18 +245,6 @@ def _get_charset_encoding(self) -> Optional[str]: return None return encoding - def _get_content(self): - """Return the internal response's content""" - return self._content - - def _set_content(self, val): - """Set the internal response's content""" - self._content = val - - def _has_content(self): - """How to check if your internal response has content""" - return self._content is not None - @property def encoding(self) -> Optional[str]: """Returns the response encoding. By default, is specified @@ -292,8 +280,8 @@ def json(self) -> Any: :rtype: any :raises json.decoder.JSONDecodeError or ValueError (in python 2.7) if object is not JSON decodable: """ - if not self._has_content(): - raise ResponseNotReadError() + # this will trigger errors if response is not read in + self.content # pylint: disable=pointless-statement if not self._json: self._json = loads(self.text) return self._json @@ -309,9 +297,9 @@ def raise_for_status(self) -> None: @property def content(self) -> bytes: """Return the response's content in bytes.""" - if not self._has_content(): + if self._content is None: raise ResponseNotReadError() - return cast(bytes, self._get_content()) + return self._content class HttpResponse(_HttpResponseBase): """**Provisional** object that represents an HTTP response. @@ -361,8 +349,8 @@ def read(self) -> bytes: :return: The read in bytes :rtype: bytes """ - if not self._has_content(): - self._set_content(b"".join(self.iter_bytes())) + if self._content is None: + self._content = b"".join(self.iter_bytes()) return self.content def iter_raw(self) -> Iterator[bytes]: @@ -443,12 +431,12 @@ async def read(self) -> bytes: :return: The response's bytes :rtype: bytes """ - if not self._has_content(): + if self._content is None: parts = [] async for part in self.iter_bytes(): # type: ignore parts.append(part) - self._set_content(b"".join(parts)) - return self._get_content() + self._content = b"".join(parts) + return self._content async def iter_raw(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process From 5be9b524186834a932b041d7cefcc08d113ddab3 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 14:54:29 -0400 Subject: [PATCH 57/64] pass self.content in aiohttp to get the decompressor --- sdk/core/azure-core/azure/core/rest/_aiohttp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/rest/_aiohttp.py b/sdk/core/azure-core/azure/core/rest/_aiohttp.py index cd09a8688cc5..b63ca4d32b4e 100644 --- a/sdk/core/azure-core/azure/core/rest/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/rest/_aiohttp.py @@ -122,7 +122,7 @@ async def iter_bytes(self) -> AsyncIterator[bytes]: async for part in iter_bytes_helper( AioHttpStreamDownloadGenerator, self, - content=self._content + content=self.content if self._content is not None else None ): yield part await self.close() From 930509a99510b50d67d71536a9a2cf76156fdb85 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 17:49:08 -0400 Subject: [PATCH 58/64] johan's comments --- sdk/core/azure-core/CHANGELOG.md | 2 +- sdk/core/azure-core/README.md | 15 +++++ .../azure-core/azure/core/_pipeline_client.py | 11 +++- .../azure/core/_pipeline_client_async.py | 13 ++++- sdk/core/azure-core/azure/core/exceptions.py | 30 +++++----- .../azure-core/azure/core/pipeline/_tools.py | 34 ++++++++++-- .../azure/core/pipeline/_tools_async.py | 34 ++++++++++++ .../azure/core/pipeline/transport/_aiohttp.py | 22 +++----- .../azure/core/pipeline/transport/_base.py | 13 ----- .../pipeline/transport/_requests_asyncio.py | 18 +++--- .../pipeline/transport/_requests_basic.py | 19 +++---- .../core/pipeline/transport/_requests_trio.py | 18 +++--- .../azure-core/azure/core/rest/__init__.py | 5 -- .../azure-core/azure/core/rest/_aiohttp.py | 4 +- .../azure/core/rest/_requests_asyncio.py | 4 +- .../azure/core/rest/_requests_basic.py | 28 +++++----- .../azure/core/rest/_requests_trio.py | 7 ++- sdk/core/azure-core/azure/core/rest/_rest.py | 26 +++++++-- .../azure-core/azure/core/rest/_rest_py3.py | 55 ++++++++++++------- .../test_rest_stream_responses.py | 12 ++-- 20 files changed, 233 insertions(+), 137 deletions(-) diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 48079c05c8e0..40b84e70df39 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -6,7 +6,7 @@ - Add new ***provisional*** methods `send_request` onto the `azure.core.PipelineClient` and `azure.core.AsyncPipelineClient`. This method takes in requests and sends them through our pipelines. -- Add new ***provisional*** module `azure.core.rest`. `azure.core.rest` is our new public simple HTTP library in `azure.core` that users will use to create requests, and consume responses. Can only be used with the provisional method `send_request` on our `PipelineClient`s +- Add new ***provisional*** module `azure.core.rest`. `azure.core.rest` is our new public simple HTTP library in `azure.core` that users will use to create requests, and consume responses. - Add new ***provisional*** errors `StreamConsumedError`, `StreamClosedError`, and `ResponseNotReadError` to `azure.core.exceptions`. These errors are thrown if you mishandle streamed responses from the provisional `azure.core.rest` module diff --git a/sdk/core/azure-core/README.md b/sdk/core/azure-core/README.md index 67c611ae596d..d098d1bf63dc 100644 --- a/sdk/core/azure-core/README.md +++ b/sdk/core/azure-core/README.md @@ -112,6 +112,21 @@ class TooManyRedirectsError(HttpResponseError): *kwargs* are keyword arguments to include with the exception. +#### **Provisional** StreamConsumedError +A **provisional** error thrown if you try to access the stream of the **provisional** +responses `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` once +the response stream has been consumed. + +#### **Provisional** StreamClosedError +A **provisional** error thrown if you try to access the stream of the **provisional** +responses `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` once +the response stream has been closed. + +#### **Provisional** ResponseNotReadError +A **provisional** error thrown if you try to access the `content` of the **provisional** +responses `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` before +reading in the response's bytes first. + ### Configurations When calling the methods, some properties can be configured by passing in as kwargs arguments. diff --git a/sdk/core/azure-core/azure/core/_pipeline_client.py b/sdk/core/azure-core/azure/core/_pipeline_client.py index cfb50d58087b..a17353671c73 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client.py @@ -40,6 +40,7 @@ RetryPolicy, ) from .pipeline.transport import RequestsTransport +from .pipeline._tools import to_rest_response as _to_rest_response try: from typing import TYPE_CHECKING @@ -189,7 +190,13 @@ def send_request(self, request, **kwargs): # type: (HTTPRequestType, Any) -> HTTPResponseType """**Provisional** method that runs the network request through the client's chained policies. - This method is marked as **provisional**, meaning it can be changed + This method is marked as **provisional**, meaning it may be changed in a future release. + + >>> from azure.core.rest import HttpRequest + >>> request = HttpRequest('GET', 'http://www.example.com') + + >>> response = client.send_request(request) + :param request: The network request you want to make. Required. :type request: ~azure.core.rest.HttpRequest @@ -202,7 +209,7 @@ def send_request(self, request, **kwargs): pipeline_response = self._pipeline.run(request_to_run, **kwargs) # pylint: disable=protected-access response = pipeline_response.http_response if rest_request: - response = response._to_rest_response() # pylint: disable=protected-access + response = _to_rest_response(response) if not kwargs.get("stream", False): response.read() response.close() diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index 9bfb96fbe654..a6fdfb2b1734 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -38,6 +38,7 @@ AsyncRetryPolicy, ) from ._pipeline_client import _prepare_request +from .pipeline._tools_async import to_rest_response as _to_rest_response try: from typing import TYPE_CHECKING, TypeVar @@ -181,7 +182,7 @@ async def _make_pipeline_call(self, request, stream, **kwargs): ) response = pipeline_response.http_response if rest_request: - rest_response = response._to_rest_response() # pylint: disable=protected-access + rest_response = _to_rest_response(response) if not stream: # in this case, the pipeline transport response already called .load_body(), so # the body is loaded. instead of doing response.read(), going to set the body @@ -204,7 +205,13 @@ def send_request( ) -> Awaitable[AsyncHTTPResponseType]: """**Provisional** method that runs the network request through the client's chained policies. - This method is marked as **provisional**, meaning it can be changed. + This method is marked as **provisional**, meaning it may be changed in a future release. + + >>> from azure.core.rest import HttpRequest + >>> request = HttpRequest('GET', 'http://www.example.com') + + >>> response = await client.send_request(request) + :param request: The network request you want to make. Required. :type request: ~azure.core.rest.HttpRequest @@ -212,6 +219,6 @@ def send_request( :return: The response of your network call. Does not do error handling on your response. :rtype: ~azure.core.rest.AsyncHttpResponse """ - from .rest import _AsyncContextManager + from .rest._rest_py3 import _AsyncContextManager wrapped = self._make_pipeline_call(request, stream=stream, **kwargs) return _AsyncContextManager(wrapped=wrapped) diff --git a/sdk/core/azure-core/azure/core/exceptions.py b/sdk/core/azure-core/azure/core/exceptions.py index e58b40274901..ba9d731baeb9 100644 --- a/sdk/core/azure-core/azure/core/exceptions.py +++ b/sdk/core/azure-core/azure/core/exceptions.py @@ -434,12 +434,12 @@ def __str__(self): return str(self._error_format) return super(ODataV4Error, self).__str__() -class StreamConsumedError(Exception): - """:bolditalic:`Provisional` error thrown if you try to access the stream of a response once consumed. +class StreamConsumedError(AzureError): + """**Provisional** error thrown if you try to access the stream of a response once consumed. - :bolditalic:`This error is provisional`, meaning it may be changed. It is thrown if you try to - read / stream an ~azure.core.rest.HttpResponse or ~azure.core.rest.AsyncHttpResponse once - the response's stream has been consumed + This error is marked as **provisional**, meaning it may be changed in a future release. It is + thrown if you try to read / stream an ~azure.core.rest.HttpResponse or + ~azure.core.rest.AsyncHttpResponse once the response's stream has been consumed. """ def __init__(self): message = ( @@ -448,12 +448,12 @@ def __init__(self): ) super(StreamConsumedError, self).__init__(message) -class StreamClosedError(Exception): - """:bolditalic:`Provisional` error thrown if you try to access the stream of a response once closed. +class StreamClosedError(AzureError): + """**Provisional** error thrown if you try to access the stream of a response once closed. - :bolditalic:`This error is provisional`, meaning it may be changed. It is thrown if you try to - read / stream an ~azure.core.rest.HttpResponse or ~azure.core.rest.AsyncHttpResponse once - the response's stream has been closed + This error is marked as **provisional**, meaning it may be changed in a future release. It is + thrown if you try to read / stream an ~azure.core.rest.HttpResponse or + ~azure.core.rest.AsyncHttpResponse once the response's stream has been closed. """ def __init__(self): message = ( @@ -462,12 +462,12 @@ def __init__(self): ) super(StreamClosedError, self).__init__(message) -class ResponseNotReadError(Exception): - """:bolditalic:`Provisional` error thrown if you try to access a response's content without reading first. +class ResponseNotReadError(AzureError): + """**Provisional** error thrown if you try to access a response's content without reading first. - :bolditalic:`This error is provisional`, meaning it may be changed. It is thrown if you try to - access an ~azure.core.rest.HttpResponse or ~azure.core.rest.AsyncHttpResponse's content without - first reading the response's bytes in first. + This error is marked as **provisional**, meaning it may be changed in a future release. It is + thrown if you try to access an ~azure.core.rest.HttpResponse or + ~azure.core.rest.AsyncHttpResponse's content without first reading the response's bytes in first. """ def __init__(self): diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools.py b/sdk/core/azure-core/azure/core/pipeline/_tools.py index 4846bcb8d7c2..a8beebd75b99 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools.py @@ -33,15 +33,39 @@ def await_result(func, *args, **kwargs): ) return result -def to_rest_response_helper(pipeline_transport_response, response_type): +def to_rest_request(pipeline_transport_request): + from ..rest import HttpRequest as RestHttpRequest + return RestHttpRequest( + method=pipeline_transport_request.method, + url=pipeline_transport_request.url, + headers=pipeline_transport_request.headers, + files=pipeline_transport_request.files, + data=pipeline_transport_request.data + ) + +def to_rest_response(pipeline_transport_response): + from .transport._requests_basic import RequestsTransportResponse + from ..rest._requests_basic import RestRequestsTransportResponse + from ..rest import HttpResponse + if isinstance(pipeline_transport_response, RequestsTransportResponse): + response_type = RestRequestsTransportResponse + else: + response_type = HttpResponse response = response_type( - request=pipeline_transport_response.request._to_rest_request(), # pylint: disable=protected-access + request=to_rest_request(pipeline_transport_response.request), internal_response=pipeline_transport_response.internal_response, ) response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access return response -def set_block_size(response): - if hasattr(response, "block_size"): +def get_block_size(response): + try: + return response._connection_data_block_size # pylint: disable=protected-access + except AttributeError: return response.block_size - return response._connection_data_block_size # pylint: disable=protected-access + +def get_internal_response(response): + try: + return response._internal_response # pylint: disable=protected-access + except AttributeError: + return response.internal_response diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index d29988bd41ee..e2cf574f9257 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -23,6 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +from ._tools import to_rest_request async def await_result(func, *args, **kwargs): """If func returns an awaitable, await it.""" @@ -31,3 +32,36 @@ async def await_result(func, *args, **kwargs): # type ignore on await: https://github.com/python/mypy/issues/7587 return await result # type: ignore return result + +def to_rest_response(pipeline_transport_response): + response_type = None + try: + from .transport import AioHttpTransportResponse + from ..rest._aiohttp import RestAioHttpTransportResponse + if isinstance(pipeline_transport_response, AioHttpTransportResponse): + response_type = RestAioHttpTransportResponse + except ImportError: + pass + try: + from .transport import AsyncioRequestsTransportResponse + from ..rest._requests_asyncio import RestAsyncioRequestsTransportResponse + if isinstance(pipeline_transport_response, AsyncioRequestsTransportResponse): + response_type = RestAsyncioRequestsTransportResponse + except ImportError: + pass + try: + from .transport import TrioRequestsTransportResponse + from ..rest._requests_trio import RestTrioRequestsTransportResponse + if isinstance(pipeline_transport_response, TrioRequestsTransportResponse): + response_type = RestTrioRequestsTransportResponse + except ImportError: + pass + if not response_type: + from ..rest import AsyncHttpResponse + response_type = AsyncHttpResponse + response = response_type( + request=to_rest_request(pipeline_transport_response.request), + internal_response=pipeline_transport_response.internal_response, + ) + response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access + return response diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 36e5dc16888c..e32d0d1c0aec 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -46,7 +46,7 @@ AsyncHttpTransport, AsyncHttpResponse, _ResponseStopIteration) -from .._tools import to_rest_response_helper, set_block_size +from .._tools import get_block_size as _get_block_size, get_internal_response as _get_internal_response # Matching requests, because why not? CONTENT_CHUNK_SIZE = 10 * 1024 @@ -216,22 +216,24 @@ def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, *, decompres self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = set_block_size(response) + self.block_size = _get_block_size(response) self._decompress = decompress - self.content_length = int(response.internal_response.headers.get('Content-Length', 0)) + internal_response = _get_internal_response(response) + self.content_length = int(internal_response.headers.get('Content-Length', 0)) self._decompressor = None def __len__(self): return self.content_length async def __anext__(self): + internal_response = _get_internal_response(self.response) try: - chunk = await self.response.internal_response.content.read(self.block_size) + chunk = await internal_response.content.read(self.block_size) if not chunk: raise _ResponseStopIteration() if not self._decompress: return chunk - enc = self.response.internal_response.headers.get('Content-Encoding') + enc = internal_response.headers.get('Content-Encoding') if not enc: return chunk enc = enc.lower() @@ -243,13 +245,13 @@ async def __anext__(self): chunk = self._decompressor.decompress(chunk) return chunk except _ResponseStopIteration: - self.response.internal_response.close() + internal_response.close() raise StopAsyncIteration() except StreamConsumedError: raise except Exception as err: _LOGGER.warning("Unable to stream download: %s", err) - self.response.internal_response.close() + internal_response.close() raise class AioHttpTransportResponse(AsyncHttpResponse): @@ -359,9 +361,3 @@ def __getstate__(self): state['internal_response'] = None # aiohttp response are not pickable (see headers comments) state['headers'] = CIMultiDict(self.headers) # MultiDictProxy is not pickable return state - - def _to_rest_response(self): - from ...rest._aiohttp import RestAioHttpTransportResponse - response = to_rest_response_helper(self, RestAioHttpTransportResponse) - response._decompress = self._decompress # pylint: disable=protected-access - return response diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_base.py b/sdk/core/azure-core/azure/core/pipeline/transport/_base.py index 31a9fa8a2d27..c807e02d841a 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_base.py @@ -473,16 +473,6 @@ def serialize(self): """ return _serialize_request(self) - def _to_rest_request(self): - from ...rest import HttpRequest as RestHttpRequest - return RestHttpRequest( - method=self.method, - url=self.url, - headers=self.headers, - files=self.files, - data=self.data - ) - class _HttpResponseBase(object): """Represent a HTTP response. @@ -587,9 +577,6 @@ def __repr__(self): type(self).__name__, self.status_code, self.reason, content_type_str ) - def _to_rest_response(self): - """Convert PipelineTransport response to a Rest response""" - class HttpResponse(_HttpResponseBase): # pylint: disable=abstract-method def stream_download(self, pipeline, **kwargs): diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index 8986fac5166c..e41e4de91325 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -44,7 +44,7 @@ _iterate_response_content) from ._requests_basic import RequestsTransportResponse, _read_raw_stream from ._base_requests_async import RequestsAsyncTransportBase -from .._tools import to_rest_response_helper, set_block_size +from .._tools import get_block_size as _get_block_size, get_internal_response as _get_internal_response _LOGGER = logging.getLogger(__name__) @@ -146,14 +146,15 @@ def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, **kwargs) -> self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = set_block_size(response) + self.block_size = _get_block_size(response) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) + internal_response = _get_internal_response(response) if decompress: - self.iter_content_func = self.response.internal_response.iter_content(self.block_size) + self.iter_content_func = internal_response.iter_content(self.block_size) else: - self.iter_content_func = _read_raw_stream(self.response.internal_response, self.block_size) + self.iter_content_func = _read_raw_stream(internal_response, self.block_size) self.content_length = int(response.headers.get('Content-Length', 0)) def __len__(self): @@ -161,6 +162,7 @@ def __len__(self): async def __anext__(self): loop = _get_running_loop() + internal_response = _get_internal_response(self.response) try: chunk = await loop.run_in_executor( None, @@ -171,13 +173,13 @@ async def __anext__(self): raise _ResponseStopIteration() return chunk except _ResponseStopIteration: - self.response.internal_response.close() + internal_response.close() raise StopAsyncIteration() except requests.exceptions.StreamConsumedError: raise except Exception as err: _LOGGER.warning("Unable to stream download: %s", err) - self.response.internal_response.close() + internal_response.close() raise @@ -187,7 +189,3 @@ class AsyncioRequestsTransportResponse(AsyncHttpResponse, RequestsTransportRespo def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]: # type: ignore """Generator for streaming request body data.""" return AsyncioStreamDownloadGenerator(pipeline, self, **kwargs) # type: ignore - - def _to_rest_response(self): - from ...rest._requests_asyncio import RestAsyncioRequestsTransportResponse - return to_rest_response_helper(self, RestAsyncioRequestsTransportResponse) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py index 4f98581a2ca9..28b81d705c16 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py @@ -46,7 +46,7 @@ _HttpResponseBase ) from ._bigger_block_size_http_adapters import BiggerBlockSizeHTTPAdapter -from .._tools import to_rest_response_helper, set_block_size +from .._tools import get_block_size as _get_block_size, get_internal_response as _get_internal_response PipelineType = TypeVar("PipelineType") @@ -73,6 +73,7 @@ def _read_raw_stream(response, chunk_size=1): yield chunk # following behavior from requests iter_content, we set content consumed to True + # https://github.com/psf/requests/blob/master/requests/models.py#L774 response._content_consumed = True # pylint: disable=protected-access class _RequestsTransportResponseBase(_HttpResponseBase): @@ -131,14 +132,15 @@ def __init__(self, pipeline, response, **kwargs): self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = set_block_size(response) + self.block_size = _get_block_size(response) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) + internal_response = _get_internal_response(response) if decompress: - self.iter_content_func = self.response.internal_response.iter_content(self.block_size) + self.iter_content_func = internal_response.iter_content(self.block_size) else: - self.iter_content_func = _read_raw_stream(self.response.internal_response, self.block_size) + self.iter_content_func = _read_raw_stream(internal_response, self.block_size) self.content_length = int(response.headers.get('Content-Length', 0)) def __len__(self): @@ -148,19 +150,20 @@ def __iter__(self): return self def __next__(self): + internal_response = _get_internal_response(self.response) try: chunk = next(self.iter_content_func) if not chunk: raise StopIteration() return chunk except StopIteration: - self.response.internal_response.close() + internal_response.close() raise StopIteration() except requests.exceptions.StreamConsumedError: raise except Exception as err: _LOGGER.warning("Unable to stream download: %s", err) - self.response.internal_response.close() + internal_response.close() raise next = __next__ # Python 2 compatibility. @@ -173,10 +176,6 @@ def stream_download(self, pipeline, **kwargs): """Generator for streaming request body data.""" return StreamDownloadGenerator(pipeline, self, **kwargs) - def _to_rest_response(self): - from ...rest._requests_basic import RestRequestsTransportResponse - return to_rest_response_helper(self, RestRequestsTransportResponse) - class RequestsTransport(HttpTransport): """Implements a basic requests HTTP sender. diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index 37262860c4c4..e21ee5115327 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -44,7 +44,7 @@ _iterate_response_content) from ._requests_basic import RequestsTransportResponse, _read_raw_stream from ._base_requests_async import RequestsAsyncTransportBase -from .._tools import to_rest_response_helper, set_block_size +from .._tools import get_block_size as _get_block_size, get_internal_response as _get_internal_response _LOGGER = logging.getLogger(__name__) @@ -62,20 +62,22 @@ def __init__(self, pipeline: Pipeline, response: AsyncHttpResponse, **kwargs) -> self.pipeline = pipeline self.request = response.request self.response = response - self.block_size = set_block_size(response) + self.block_size = _get_block_size(response) decompress = kwargs.pop("decompress", True) if len(kwargs) > 0: raise TypeError("Got an unexpected keyword argument: {}".format(list(kwargs.keys())[0])) + internal_response = _get_internal_response(response) if decompress: - self.iter_content_func = self.response.internal_response.iter_content(self.block_size) + self.iter_content_func = internal_response.iter_content(self.block_size) else: - self.iter_content_func = _read_raw_stream(self.response.internal_response, self.block_size) + self.iter_content_func = _read_raw_stream(internal_response, self.block_size) self.content_length = int(response.headers.get('Content-Length', 0)) def __len__(self): return self.content_length async def __anext__(self): + internal_response = _get_internal_response(self.response) try: try: chunk = await trio.to_thread.run_sync( @@ -91,13 +93,13 @@ async def __anext__(self): raise _ResponseStopIteration() return chunk except _ResponseStopIteration: - self.response.internal_response.close() + internal_response.close() raise StopAsyncIteration() except requests.exceptions.StreamConsumedError: raise except Exception as err: _LOGGER.warning("Unable to stream download: %s", err) - self.response.internal_response.close() + internal_response.close() raise class TrioRequestsTransportResponse(AsyncHttpResponse, RequestsTransportResponse): # type: ignore @@ -108,10 +110,6 @@ def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]: # ty """ return TrioStreamDownloadGenerator(pipeline, self, **kwargs) - def _to_rest_response(self): - from ...rest._requests_trio import RestTrioRequestsTransportResponse - return to_rest_response_helper(self, RestTrioRequestsTransportResponse) - class TrioRequestsTransport(RequestsAsyncTransportBase): # type: ignore """Identical implementation as the synchronous RequestsTransport wrapped in a class with diff --git a/sdk/core/azure-core/azure/core/rest/__init__.py b/sdk/core/azure-core/azure/core/rest/__init__.py index ed79c71b858c..2fc73b837f14 100644 --- a/sdk/core/azure-core/azure/core/rest/__init__.py +++ b/sdk/core/azure-core/azure/core/rest/__init__.py @@ -27,29 +27,24 @@ from ._rest_py3 import ( HttpRequest, HttpResponse, - _HttpResponseBase, ) except (SyntaxError, ImportError): from ._rest import ( # type: ignore HttpRequest, HttpResponse, - _HttpResponseBase, ) __all__ = [ "HttpRequest", "HttpResponse", - "_HttpResponseBase", ] try: from ._rest_py3 import ( # pylint: disable=unused-import AsyncHttpResponse, - _AsyncContextManager, ) __all__.extend([ "AsyncHttpResponse", - "_AsyncContextManager", ]) except (SyntaxError, ImportError): diff --git a/sdk/core/azure-core/azure/core/rest/_aiohttp.py b/sdk/core/azure-core/azure/core/rest/_aiohttp.py index b63ca4d32b4e..0d78c9f33527 100644 --- a/sdk/core/azure-core/azure/core/rest/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/rest/_aiohttp.py @@ -107,6 +107,7 @@ def text(self) -> str: async def iter_raw(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process + :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ @@ -116,6 +117,7 @@ async def iter_raw(self) -> AsyncIterator[bytes]: async def iter_bytes(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process + :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ @@ -141,5 +143,5 @@ async def close(self) -> None: :rtype: None """ self.is_closed = True - self.internal_response.close() + self._internal_response.close() await asyncio.sleep(0) diff --git a/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py b/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py index 98986ec2b07f..b21545a79804 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_asyncio.py @@ -66,7 +66,7 @@ async def close(self) -> None: :rtype: None """ self.is_closed = True - self.internal_response.close() + self._internal_response.close() await asyncio.sleep(0) async def read(self) -> bytes: @@ -79,5 +79,5 @@ async def read(self) -> bytes: parts = [] async for part in self.iter_bytes(): # type: ignore parts.append(part) - self.internal_response._content = b"".join(parts) # pylint: disable=protected-access + self._internal_response._content = b"".join(parts) # pylint: disable=protected-access return self.content diff --git a/sdk/core/azure-core/azure/core/rest/_requests_basic.py b/sdk/core/azure-core/azure/core/rest/_requests_basic.py index 935cc545f78c..92e71b1e155c 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_basic.py @@ -26,7 +26,7 @@ from typing import TYPE_CHECKING from ..exceptions import ResponseNotReadError, StreamConsumedError, StreamClosedError -from . import _HttpResponseBase, HttpResponse +from ._rest import _HttpResponseBase, HttpResponse from ..pipeline.transport._requests_basic import StreamDownloadGenerator if TYPE_CHECKING: @@ -42,40 +42,40 @@ def _has_content(response): class _RestRequestsTransportResponseBase(_HttpResponseBase): def __init__(self, **kwargs): super(_RestRequestsTransportResponseBase, self).__init__(**kwargs) - self.status_code = self.internal_response.status_code - self.headers = self.internal_response.headers - self.reason = self.internal_response.reason - self.content_type = self.internal_response.headers.get('content-type') + self.status_code = self._internal_response.status_code + self.headers = self._internal_response.headers + self.reason = self._internal_response.reason + self.content_type = self._internal_response.headers.get('content-type') @property def content(self): # type: () -> bytes - if not self.internal_response._content_consumed: # pylint: disable=protected-access + if not self._internal_response._content_consumed: # pylint: disable=protected-access # if we just call .content, requests will read in the content. # we want to read it in our own way raise ResponseNotReadError() try: - return self.internal_response.content + return self._internal_response.content except RuntimeError: # requests throws a RuntimeError if the content for a response is already consumed raise ResponseNotReadError() def _get_content(self): """Return the internal response's content""" - if not self.internal_response._content_consumed: # pylint: disable=protected-access + if not self._internal_response._content_consumed: # pylint: disable=protected-access # if we just call .content, requests will read in the content. # we want to read it in our own way return None try: - return self.internal_response.content + return self._internal_response.content except RuntimeError: # requests throws a RuntimeError if the content for a response is already consumed return None def _set_content(self, val): """Set the internal response's content""" - self.internal_response._content = val # pylint: disable=protected-access + self._internal_response._content = val # pylint: disable=protected-access @_HttpResponseBase.encoding.setter # type: ignore def encoding(self, value): @@ -89,18 +89,18 @@ def encoding(self, value): # - https://github.com/psf/requests/issues/1737 # - https://github.com/psf/requests/issues/2086 from codecs import BOM_UTF8 - if self.internal_response.content[:3] == BOM_UTF8: + if self._internal_response.content[:3] == BOM_UTF8: encoding = "utf-8-sig" if encoding: if encoding == "utf-8": encoding = "utf-8-sig" - self.internal_response.encoding = encoding + self._internal_response.encoding = encoding @property def text(self): # this will trigger errors if response is not read in self.content # pylint: disable=pointless-statement - return self.internal_response.text + return self._internal_response.text def _stream_download_helper(decompress, response): if response.is_stream_consumed: @@ -157,5 +157,5 @@ def read(self): :rtype: bytes """ if not _has_content(self): - self.internal_response._content = b"".join(self.iter_bytes()) # pylint: disable=protected-access + self._internal_response._content = b"".join(self.iter_bytes()) # pylint: disable=protected-access return self.content diff --git a/sdk/core/azure-core/azure/core/rest/_requests_trio.py b/sdk/core/azure-core/azure/core/rest/_requests_trio.py index 9834bff51def..bc9ae98206e2 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_trio.py @@ -33,8 +33,10 @@ class RestTrioRequestsTransportResponse(AsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore """Asynchronous streaming of data from the response. """ + async def iter_raw(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process + :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ @@ -44,6 +46,7 @@ async def iter_raw(self) -> AsyncIterator[bytes]: async def iter_bytes(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process + :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ @@ -66,10 +69,10 @@ async def read(self) -> bytes: parts = [] async for part in self.iter_bytes(): # type: ignore parts.append(part) - self.internal_response._content = b"".join(parts) # pylint: disable=protected-access + self._internal_response._content = b"".join(parts) # pylint: disable=protected-access return self.content async def close(self) -> None: self.is_closed = True - self.internal_response.close() + self._internal_response.close() await trio.sleep(0) diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index 5d7f366a2de2..a5924eb27de1 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -67,7 +67,15 @@ class HttpRequest(object): """Provisional object that represents an HTTP request. - **This object is provisional**, meaning it may be changed. + **This object is provisional**, meaning it may be changed in a future release. + + It should be passed to your client's `send_request` method. + + >>> from azure.core.rest import HttpRequest + >>> request = HttpRequest('GET', 'http://www.example.com') + + >>> response = client.send_request(request) + :param str method: HTTP method (GET, HEAD, etc.) :param str url: The url for your request @@ -188,7 +196,7 @@ class _HttpResponseBase(object): # pylint: disable=too-many-instance-attributes def __init__(self, **kwargs): # type: (Any) -> None self.request = kwargs.pop("request") - self.internal_response = kwargs.pop("internal_response") + self._internal_response = kwargs.pop("internal_response") self.status_code = None self.headers = {} # type: HeadersType self.reason = None @@ -293,11 +301,19 @@ def __repr__(self): class HttpResponse(_HttpResponseBase): # pylint: disable=too-many-instance-attributes """**Provisional** object that represents an HTTP response. - **This object is provisional**, meaning it may be changed. + **This object is provisional**, meaning it may be changed in a future release. + + It is returned from your client's `send_request` method if you pass in + an :class:`~azure.core.rest.HttpRequest` + + >>> from azure.core.rest import HttpRequest + >>> request = HttpRequest('GET', 'http://www.example.com') + + >>> response = client.send_request(request) + :keyword request: The request that resulted in this response. :paramtype request: ~azure.core.rest.HttpRequest - :keyword internal_response: The object returned from the HTTP library. :ivar int status_code: The status code of this response :ivar mapping headers: The response headers :ivar str reason: The reason phrase for this response @@ -324,7 +340,7 @@ def __enter__(self): def close(self): # type: (...) -> None self.is_closed = True - self.internal_response.close() + self._internal_response.close() def __exit__(self, *args): # type: (...) -> None diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index bac206a80c53..bccb5b0fb9ca 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -87,7 +87,15 @@ async def close(self): class HttpRequest: """**Provisional** object that represents an HTTP request. - **This object is provisional**, meaning it may be changed. + **This object is provisional**, meaning it may be changed in a future release. + + It should be passed to your client's `send_request` method. + + >>> from azure.core.rest import HttpRequest + >>> request = HttpRequest('GET', 'http://www.example.com') + + >>> response = client.send_request(request) + :param str method: HTTP method (GET, HEAD, etc.) :param str url: The url for your request @@ -214,10 +222,10 @@ def __init__( self, *, request: HttpRequest, - internal_response, + **kwargs ): self.request = request - self.internal_response = internal_response + self._internal_response = kwargs.pop("internal_response") self.status_code = None self.headers = {} # type: HeadersType self.reason = None @@ -304,11 +312,19 @@ def content(self) -> bytes: class HttpResponse(_HttpResponseBase): """**Provisional** object that represents an HTTP response. - **This object is provisional**, meaning it may be changed. + **This object is provisional**, meaning it may be changed in a future release. + + It is returned from your client's `send_request` method if you pass in + an :class:`~azure.core.rest.HttpRequest` + + >>> from azure.core.rest import HttpRequest + >>> request = HttpRequest('GET', 'http://www.example.com') + + >>> response = client.send_request(request) + :keyword request: The request that resulted in this response. :paramtype request: ~azure.core.rest.HttpRequest - :keyword internal_response: The object returned from the HTTP library. :ivar int status_code: The status code of this response :ivar mapping headers: The response headers :ivar str reason: The reason phrase for this response @@ -338,7 +354,7 @@ def close(self) -> None: :rtype: None """ self.is_closed = True - self.internal_response.close() + self._internal_response.close() def __exit__(self, *args) -> None: self.close() @@ -401,7 +417,16 @@ def __repr__(self) -> str: class AsyncHttpResponse(_HttpResponseBase): """**Provisional** object that represents an Async HTTP response. - **This object is provisional**, meaning it may be changed. + **This object is provisional**, meaning it may be changed in a future release. + + It is returned from your async client's `send_request` method if you pass in + an :class:`~azure.core.rest.HttpRequest` + + >>> from azure.core.rest import HttpRequest + >>> request = HttpRequest('GET', 'http://www.example.com') + + >>> response = await client.send_request(request) + :keyword request: The request that resulted in this response. :paramtype request: ~azure.core.rest.HttpRequest @@ -444,13 +469,8 @@ async def iter_raw(self) -> AsyncIterator[bytes]: :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - # If you don't have a yield in an AsyncIterator function, - # mypy will think it's a coroutine - # see here https://github.com/python/mypy/issues/5385#issuecomment-407281656 - # So, adding this weird yield thing - for _ in []: - yield _ raise NotImplementedError() + yield # getting around mypy behavior, see https://github.com/python/mypy/issues/10732 async def iter_bytes(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process @@ -458,13 +478,8 @@ async def iter_bytes(self) -> AsyncIterator[bytes]: :return: An async iterator of bytes from the response :rtype: AsyncIterator[bytes] """ - # If you don't have a yield in an AsyncIterator function, - # mypy will think it's a coroutine - # see here https://github.com/python/mypy/issues/5385#issuecomment-407281656 - # So, adding this weird yield thing - for _ in []: - yield _ raise NotImplementedError() + yield # getting around mypy behavior, see https://github.com/python/mypy/issues/10732 async def iter_text(self) -> AsyncIterator[str]: """Asynchronously iterates over the text in the response. @@ -494,7 +509,7 @@ async def close(self) -> None: :rtype: None """ self.is_closed = True - await self.internal_response.close() + await self._internal_response.close() async def __aexit__(self, *args) -> None: await self.close() diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py index 3b0e6469e948..0c52c832da58 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py @@ -13,7 +13,7 @@ def _assert_stream_state(response, open): # if open is true, check the stream is open. # if false, check if everything is closed checks = [ - response.internal_response._content_consumed, + response._internal_response._content_consumed, response.is_closed, response.is_stream_consumed ] @@ -27,12 +27,12 @@ def test_iter_raw(client): with client.send_request(request, stream=True) as response: raw = b"" for part in response.iter_raw(): - assert not response.internal_response._content_consumed + assert not response._internal_response._content_consumed assert not response.is_closed assert response.is_stream_consumed # we follow httpx behavior here raw += part assert raw == b"Hello, world!" - assert response.internal_response._content_consumed + assert response._internal_response._content_consumed assert response.is_closed assert response.is_stream_consumed @@ -79,11 +79,11 @@ def test_iter_bytes(client): with client.send_request(request, stream=True) as response: raw = b"" for chunk in response.iter_bytes(): - assert not response.internal_response._content_consumed + assert not response._internal_response._content_consumed assert not response.is_closed assert response.is_stream_consumed # we follow httpx behavior here raw += chunk - assert response.internal_response._content_consumed + assert response._internal_response._content_consumed assert response.is_closed assert response.is_stream_consumed assert raw == b"Hello, world!" @@ -233,4 +233,4 @@ def test_error_reading(client): response.raise_for_status() response.read() assert response.content == b"" - response.content + # try giving a really slow response, see what happens From 7fd947f0f6e0fd2c620b440808ffcba38b6477cb Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 18:09:20 -0400 Subject: [PATCH 59/64] stop importing resposne types once we get one --- .../azure/core/pipeline/_tools_async.py | 17 +++++++++-------- .../azure/core/rest/_requests_trio.py | 1 - 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index e2cf574f9257..de59dfdd86ed 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -33,32 +33,33 @@ async def await_result(func, *args, **kwargs): return await result # type: ignore return result -def to_rest_response(pipeline_transport_response): - response_type = None +def _get_response_type(pipeline_transport_response): try: from .transport import AioHttpTransportResponse from ..rest._aiohttp import RestAioHttpTransportResponse if isinstance(pipeline_transport_response, AioHttpTransportResponse): - response_type = RestAioHttpTransportResponse + return RestAioHttpTransportResponse except ImportError: pass try: from .transport import AsyncioRequestsTransportResponse from ..rest._requests_asyncio import RestAsyncioRequestsTransportResponse if isinstance(pipeline_transport_response, AsyncioRequestsTransportResponse): - response_type = RestAsyncioRequestsTransportResponse + return RestAsyncioRequestsTransportResponse except ImportError: pass try: from .transport import TrioRequestsTransportResponse from ..rest._requests_trio import RestTrioRequestsTransportResponse if isinstance(pipeline_transport_response, TrioRequestsTransportResponse): - response_type = RestTrioRequestsTransportResponse + return RestTrioRequestsTransportResponse except ImportError: pass - if not response_type: - from ..rest import AsyncHttpResponse - response_type = AsyncHttpResponse + from ..rest import AsyncHttpResponse + return AsyncHttpResponse + +def to_rest_response(pipeline_transport_response): + response_type = _get_response_type(pipeline_transport_response) response = response_type( request=to_rest_request(pipeline_transport_response.request), internal_response=pipeline_transport_response.internal_response, diff --git a/sdk/core/azure-core/azure/core/rest/_requests_trio.py b/sdk/core/azure-core/azure/core/rest/_requests_trio.py index bc9ae98206e2..9806380ef04f 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_trio.py @@ -33,7 +33,6 @@ class RestTrioRequestsTransportResponse(AsyncHttpResponse, _RestRequestsTransportResponseBase): # type: ignore """Asynchronous streaming of data from the response. """ - async def iter_raw(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process From 931aba3cd37c6b568dd333409aa9370744eb1129 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 18:12:41 -0400 Subject: [PATCH 60/64] set decompress property for aiohttp in to_rest_response --- sdk/core/azure-core/azure/core/pipeline/_tools_async.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index de59dfdd86ed..9da8a6b1bddf 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -64,5 +64,7 @@ def to_rest_response(pipeline_transport_response): request=to_rest_request(pipeline_transport_response.request), internal_response=pipeline_transport_response.internal_response, ) + if hasattr(pipeline_transport_response, "_decompress"): + response._decompress = pipeline_transport_response._decompress response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access return response From 27f4af968bca59ec040c56ea4b88e4de04cdc05c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 19:16:09 -0400 Subject: [PATCH 61/64] remove special content from aiohttp and add reuest info to new rest errors --- .../azure/core/_pipeline_client_async.py | 6 ++--- sdk/core/azure-core/azure/core/exceptions.py | 21 ++++++++++------ .../azure/core/pipeline/_tools_async.py | 2 -- .../azure-core/azure/core/rest/_aiohttp.py | 25 +------------------ .../azure/core/rest/_helpers_py3.py | 10 +++++--- .../azure/core/rest/_requests_basic.py | 14 ++++++----- sdk/core/azure-core/azure/core/rest/_rest.py | 4 +-- .../azure-core/azure/core/rest/_rest_py3.py | 10 +++++--- .../test_rest_stream_responses_async.py | 11 +++++--- .../test_rest_stream_responses.py | 11 +++++--- 10 files changed, 53 insertions(+), 61 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index a6fdfb2b1734..6541b2e240cf 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -174,16 +174,16 @@ def _build_pipeline(self, config, **kwargs): # pylint: disable=no-self-use return AsyncPipeline(transport, policies) - async def _make_pipeline_call(self, request, stream, **kwargs): + async def _make_pipeline_call(self, request, **kwargs): rest_request, request_to_run = _prepare_request(request) return_pipeline_response = kwargs.pop("_return_pipeline_response", False) pipeline_response = await self._pipeline.run( - request_to_run, stream=stream, **kwargs # pylint: disable=protected-access + request_to_run, **kwargs # pylint: disable=protected-access ) response = pipeline_response.http_response if rest_request: rest_response = _to_rest_response(response) - if not stream: + if not kwargs.get("stream"): # in this case, the pipeline transport response already called .load_body(), so # the body is loaded. instead of doing response.read(), going to set the body # to the internal content diff --git a/sdk/core/azure-core/azure/core/exceptions.py b/sdk/core/azure-core/azure/core/exceptions.py index ba9d731baeb9..0f827008a717 100644 --- a/sdk/core/azure-core/azure/core/exceptions.py +++ b/sdk/core/azure-core/azure/core/exceptions.py @@ -441,10 +441,12 @@ class StreamConsumedError(AzureError): thrown if you try to read / stream an ~azure.core.rest.HttpResponse or ~azure.core.rest.AsyncHttpResponse once the response's stream has been consumed. """ - def __init__(self): + def __init__(self, response): message = ( - "You are attempting to read or stream content that has already been streamed. " - "You have likely already consumed this stream, so it can not be accessed anymore." + "You are attempting to read or stream the content from request {}. "\ + "You have likely already consumed this stream, so it can not be accessed anymore.".format( + response.request + ) ) super(StreamConsumedError, self).__init__(message) @@ -455,10 +457,10 @@ class StreamClosedError(AzureError): thrown if you try to read / stream an ~azure.core.rest.HttpResponse or ~azure.core.rest.AsyncHttpResponse once the response's stream has been closed. """ - def __init__(self): + def __init__(self, response): message = ( - "The response's content can no longer be read or streamed, since the " - "response has already been closed." + "The content for response from request {} can no longer be read or streamed, since the "\ + "response has already been closed.".format(response.request) ) super(StreamClosedError, self).__init__(message) @@ -470,8 +472,11 @@ class ResponseNotReadError(AzureError): ~azure.core.rest.AsyncHttpResponse's content without first reading the response's bytes in first. """ - def __init__(self): + def __init__(self, response): message = ( - "You have not read in the response's bytes yet. Call response.read() first." + "You have not read in the bytes for the response from request {}. "\ + "Call .read() on the response first.".format( + response.request + ) ) super(ResponseNotReadError, self).__init__(message) diff --git a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py index 9da8a6b1bddf..de59dfdd86ed 100644 --- a/sdk/core/azure-core/azure/core/pipeline/_tools_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/_tools_async.py @@ -64,7 +64,5 @@ def to_rest_response(pipeline_transport_response): request=to_rest_request(pipeline_transport_response.request), internal_response=pipeline_transport_response.internal_response, ) - if hasattr(pipeline_transport_response, "_decompress"): - response._decompress = pipeline_transport_response._decompress response._connection_data_block_size = pipeline_transport_response.block_size # pylint: disable=protected-access return response diff --git a/sdk/core/azure-core/azure/core/rest/_aiohttp.py b/sdk/core/azure-core/azure/core/rest/_aiohttp.py index 0d78c9f33527..246238aa26b2 100644 --- a/sdk/core/azure-core/azure/core/rest/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/rest/_aiohttp.py @@ -36,7 +36,6 @@ import chardet # type: ignore from ._helpers_py3 import iter_raw_helper, iter_bytes_helper from ..pipeline.transport._aiohttp import AioHttpStreamDownloadGenerator -from ..exceptions import ResponseNotReadError class RestAioHttpTransportResponse(AsyncHttpResponse): @@ -51,28 +50,6 @@ def __init__( self.headers = CIMultiDict(internal_response.headers) # type: ignore self.reason = internal_response.reason self.content_type = internal_response.headers.get('content-type') - self._decompress = None - self._decompressed_content = None - - @property - def content(self) -> bytes: - if self._content is None: - raise ResponseNotReadError() - if not self._decompress: - return self._content - enc = self.headers.get('Content-Encoding') - if not enc: - return self._content - enc = enc.lower() - if enc in ("gzip", "deflate"): - if self._decompressed_content: - return self._decompressed_content - import zlib - zlib_mode = 16 + zlib.MAX_WBITS if enc == "gzip" else zlib.MAX_WBITS - decompressor = zlib.decompressobj(wbits=zlib_mode) - self._decompressed_content = decompressor.decompress(self._content) - return self._decompressed_content - return self._content @property def text(self) -> str: @@ -124,7 +101,7 @@ async def iter_bytes(self) -> AsyncIterator[bytes]: async for part in iter_bytes_helper( AioHttpStreamDownloadGenerator, self, - content=self.content if self._content is not None else None + content=self._content ): yield part await self.close() diff --git a/sdk/core/azure-core/azure/core/rest/_helpers_py3.py b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py index 5c4efd78f213..4c09b3dbe1ab 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py @@ -61,9 +61,9 @@ def _stream_download_helper( response, ) -> AsyncIteratorType[bytes]: if response.is_stream_consumed: - raise StreamConsumedError() + raise StreamConsumedError(response) if response.is_closed: - raise StreamClosedError() + raise StreamClosedError(response) response.is_stream_consumed = True return stream_download_generator( @@ -77,8 +77,10 @@ async def iter_bytes_helper( response, content: Optional[bytes], ) -> AsyncIteratorType[bytes]: - if content: # pylint: disable=protected-access - yield content # pylint: disable=protected-access + if content: + chunk_size = response._connection_data_block_size # pylint: disable=protected-access + for i in range(0, len(content), chunk_size): + yield content[i : i + chunk_size] else: async for part in _stream_download_helper( decompress=True, diff --git a/sdk/core/azure-core/azure/core/rest/_requests_basic.py b/sdk/core/azure-core/azure/core/rest/_requests_basic.py index 92e71b1e155c..0e51e32bac57 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_basic.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from ..exceptions import ResponseNotReadError, StreamConsumedError, StreamClosedError from ._rest import _HttpResponseBase, HttpResponse @@ -53,13 +53,13 @@ def content(self): if not self._internal_response._content_consumed: # pylint: disable=protected-access # if we just call .content, requests will read in the content. # we want to read it in our own way - raise ResponseNotReadError() + raise ResponseNotReadError(self) try: return self._internal_response.content except RuntimeError: # requests throws a RuntimeError if the content for a response is already consumed - raise ResponseNotReadError() + raise ResponseNotReadError(self) def _get_content(self): """Return the internal response's content""" @@ -104,9 +104,9 @@ def text(self): def _stream_download_helper(decompress, response): if response.is_stream_consumed: - raise StreamConsumedError() + raise StreamConsumedError(response) if response.is_closed: - raise StreamClosedError() + raise StreamClosedError(response) response.is_stream_consumed = True stream_download = StreamDownloadGenerator( @@ -127,7 +127,9 @@ def iter_bytes(self): :rtype: Iterator[str] """ if _has_content(self): - yield self.content + chunk_size = cast(int, self._connection_data_block_size) + for i in range(0, len(self.content), chunk_size): + yield self.content[i : i + chunk_size] else: for part in _stream_download_helper( decompress=True, diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index a5924eb27de1..02f5468aaa45 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -205,7 +205,7 @@ def __init__(self, **kwargs): self._num_bytes_downloaded = 0 self.content_type = None self._json = None # this is filled in ContentDecodePolicy, when we deserialize - self._connection_data_block_size = None + self._connection_data_block_size = None # type: Optional[int] self._content = None # type: Optional[bytes] @property @@ -286,7 +286,7 @@ def content(self): # type: (...) -> bytes """Return the response's content in bytes.""" if self._content is None: - raise ResponseNotReadError() + raise ResponseNotReadError(self) return self._content def __repr__(self): diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index bccb5b0fb9ca..532ab323a811 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -306,7 +306,7 @@ def raise_for_status(self) -> None: def content(self) -> bytes: """Return the response's content in bytes.""" if self._content is None: - raise ResponseNotReadError() + raise ResponseNotReadError(self) return self._content class HttpResponse(_HttpResponseBase): @@ -458,7 +458,7 @@ async def read(self) -> bytes: """ if self._content is None: parts = [] - async for part in self.iter_bytes(): # type: ignore + async for part in self.iter_bytes(): parts.append(part) self._content = b"".join(parts) return self._content @@ -470,7 +470,8 @@ async def iter_raw(self) -> AsyncIterator[bytes]: :rtype: AsyncIterator[bytes] """ raise NotImplementedError() - yield # getting around mypy behavior, see https://github.com/python/mypy/issues/10732 + # getting around mypy behavior, see https://github.com/python/mypy/issues/10732 + yield # pylint: disable=unreachable async def iter_bytes(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will decompress in the process @@ -479,7 +480,8 @@ async def iter_bytes(self) -> AsyncIterator[bytes]: :rtype: AsyncIterator[bytes] """ raise NotImplementedError() - yield # getting around mypy behavior, see https://github.com/python/mypy/issues/10732 + # getting around mypy behavior, see https://github.com/python/mypy/issues/10732 + yield # pylint: disable=unreachable async def iter_text(self) -> AsyncIterator[str]: """Asynchronously iterates over the text in the response. diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py index c41f7f290818..b4968ff71cd9 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py @@ -115,7 +115,7 @@ async def test_streaming_response(client): assert response.is_closed @pytest.mark.asyncio -async def test_cannot_read_after_stream_consumed(client): +async def test_cannot_read_after_stream_consumed(port, client): request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: content = b"" @@ -124,17 +124,20 @@ async def test_cannot_read_after_stream_consumed(client): with pytest.raises(StreamConsumedError) as ex: await response.read() - assert "You are attempting to read or stream content that has already been streamed" in str(ex.value) + assert "".format(port) in str(ex.value) + assert "You have likely already consumed this stream, so it can not be accessed anymore" in str(ex.value) + @pytest.mark.asyncio -async def test_cannot_read_after_response_closed(client): +async def test_cannot_read_after_response_closed(port, client): request = HttpRequest("GET", "/streams/basic") async with client.send_request(request, stream=True) as response: pass with pytest.raises(StreamClosedError) as ex: await response.read() - assert "The response's content can no longer be read or streamed, since the response has already been closed." in str(ex.value) + assert "".format(port) in str(ex.value) + assert "can no longer be read or streamed, since the response has already been closed" in str(ex.value) @pytest.mark.asyncio async def test_decompress_plain_no_header(client): diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py index 0c52c832da58..209c12f2fdc6 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py @@ -119,7 +119,7 @@ def test_sync_streaming_response(client): assert response.content == b"Hello, world!" assert response.is_closed -def test_cannot_read_after_stream_consumed(client): +def test_cannot_read_after_stream_consumed(client, port): request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: @@ -132,16 +132,19 @@ def test_cannot_read_after_stream_consumed(client): with pytest.raises(StreamConsumedError) as ex: response.read() - assert "You are attempting to read or stream content that has already been streamed." in str(ex.value) + assert "".format(port) in str(ex.value) + assert "You have likely already consumed this stream, so it can not be accessed anymore" in str(ex.value) -def test_cannot_read_after_response_closed(client): +def test_cannot_read_after_response_closed(port, client): request = HttpRequest("GET", "/streams/basic") with client.send_request(request, stream=True) as response: response.close() with pytest.raises(StreamClosedError) as ex: response.read() - assert "The response's content can no longer be read or streamed, since the response has already been closed." in str(ex.value) + # breaking up assert into multiple lines + assert "".format(port) in str(ex.value) + assert "can no longer be read or streamed, since the response has already been closed" in str(ex.value) def test_decompress_plain_no_header(client): # thanks to Xiang Yan for this test! From 9d4013bbe44f4b4adc3920fc9a0fda6cdb3255e6 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 20:16:13 -0400 Subject: [PATCH 62/64] default application/json to utf-8, and use chardet to detect endoing as well --- .../azure-core/azure/core/rest/_aiohttp.py | 37 --------------- .../azure-core/azure/core/rest/_helpers.py | 27 +++++++++++ .../azure/core/rest/_requests_basic.py | 47 +++++++------------ sdk/core/azure-core/azure/core/rest/_rest.py | 17 +------ .../azure-core/azure/core/rest/_rest_py3.py | 17 ++----- .../test_rest_http_response_async.py | 6 +-- .../test_rest_http_response.py | 6 +-- 7 files changed, 56 insertions(+), 101 deletions(-) diff --git a/sdk/core/azure-core/azure/core/rest/_aiohttp.py b/sdk/core/azure-core/azure/core/rest/_aiohttp.py index 246238aa26b2..f25d9f7679b0 100644 --- a/sdk/core/azure-core/azure/core/rest/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/rest/_aiohttp.py @@ -25,15 +25,9 @@ # -------------------------------------------------------------------------- import asyncio -import codecs from typing import AsyncIterator from multidict import CIMultiDict -import aiohttp from . import HttpRequest, AsyncHttpResponse -try: - import cchardet as chardet -except ImportError: # pragma: no cover - import chardet # type: ignore from ._helpers_py3 import iter_raw_helper, iter_bytes_helper from ..pipeline.transport._aiohttp import AioHttpStreamDownloadGenerator @@ -51,37 +45,6 @@ def __init__( self.reason = internal_response.reason self.content_type = internal_response.headers.get('content-type') - @property - def text(self) -> str: - content = self.content - encoding = self.encoding - ctype = self.headers.get(aiohttp.hdrs.CONTENT_TYPE, "").lower() - mimetype = aiohttp.helpers.parse_mimetype(ctype) - - encoding = mimetype.parameters.get("charset") - if encoding: - try: - codecs.lookup(encoding) - except LookupError: - encoding = None - if not encoding: - if mimetype.type == "application" and ( - mimetype.subtype == "json" or mimetype.subtype == "rdap" - ): - # RFC 7159 states that the default encoding is UTF-8. - # RFC 7483 defines application/rdap+json - encoding = "utf-8" - elif content is None: - raise RuntimeError( - "Cannot guess the encoding of a not yet read content" - ) - else: - encoding = chardet.detect(content)["encoding"] - if not encoding: - encoding = "utf-8-sig" - - return content.decode(encoding) - async def iter_raw(self) -> AsyncIterator[bytes]: """Asynchronously iterates over the response's bytes. Will not decompress in the process diff --git a/sdk/core/azure-core/azure/core/rest/_helpers.py b/sdk/core/azure-core/azure/core/rest/_helpers.py index 5caacb500265..27e11299fdb5 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers.py @@ -25,6 +25,7 @@ # -------------------------------------------------------------------------- import os import codecs +import cgi from enum import Enum from json import dumps import collections @@ -49,6 +50,11 @@ from urlparse import urlparse # type: ignore except ImportError: from urllib.parse import urlparse +try: + import cchardet as chardet +except ImportError: # pragma: no cover + import chardet # type: ignore +from ..exceptions import ResponseNotReadError ################################### TYPES SECTION ######################### @@ -277,3 +283,24 @@ def from_pipeline_transport_request_helper(request_class, pipeline_transport_req files=pipeline_transport_request.files, data=pipeline_transport_request.data ) + +def get_charset_encoding(response): + content_type = response.headers.get("Content-Type") + + if not content_type: + return None + _, params = cgi.parse_header(content_type) + encoding = params.get('charset') # -> utf-8 + if encoding is None: + if content_type in ("application/json", "application/rdap+json"): + # RFC 7159 states that the default encoding is UTF-8. + # RFC 7483 defines application/rdap+json + encoding = "utf-8" + else: + try: + encoding = chardet.detect(response.content)["encoding"] + except ResponseNotReadError: + pass + if encoding is None or not lookup_encoding(encoding): + return None + return encoding diff --git a/sdk/core/azure-core/azure/core/rest/_requests_basic.py b/sdk/core/azure-core/azure/core/rest/_requests_basic.py index 0e51e32bac57..8bf83ad4e835 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_basic.py @@ -30,7 +30,7 @@ from ..pipeline.transport._requests_basic import StreamDownloadGenerator if TYPE_CHECKING: - from typing import Iterator + from typing import Iterator, Optional def _has_content(response): try: @@ -61,40 +61,29 @@ def content(self): # requests throws a RuntimeError if the content for a response is already consumed raise ResponseNotReadError(self) - def _get_content(self): - """Return the internal response's content""" - if not self._internal_response._content_consumed: # pylint: disable=protected-access - # if we just call .content, requests will read in the content. - # we want to read it in our own way - return None - try: - return self._internal_response.content - except RuntimeError: - # requests throws a RuntimeError if the content for a response is already consumed - return None - - def _set_content(self, val): - """Set the internal response's content""" - self._internal_response._content = val # pylint: disable=protected-access - - @_HttpResponseBase.encoding.setter # type: ignore - def encoding(self, value): - # type: (str) -> None - # ignoring setter bc of known mypy issue https://github.com/python/mypy/issues/1465 - self._encoding = value - encoding = value - if not encoding: + @property + def encoding(self): + # type: () -> Optional[str] + retval = super(_RestRequestsTransportResponseBase, self).encoding + if not retval: # There is a few situation where "requests" magic doesn't fit us: # - https://github.com/psf/requests/issues/654 # - https://github.com/psf/requests/issues/1737 # - https://github.com/psf/requests/issues/2086 from codecs import BOM_UTF8 if self._internal_response.content[:3] == BOM_UTF8: - encoding = "utf-8-sig" - if encoding: - if encoding == "utf-8": - encoding = "utf-8-sig" - self._internal_response.encoding = encoding + retval = "utf-8-sig" + if retval: + if retval == "utf-8": + retval = "utf-8-sig" + return retval + + @encoding.setter # type: ignore + def encoding(self, value): + # type: (str) -> None + # ignoring setter bc of known mypy issue https://github.com/python/mypy/issues/1465 + self._encoding = value + self._internal_response.encoding = value @property def text(self): diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index 02f5468aaa45..e9345eef01b3 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -24,7 +24,6 @@ # # -------------------------------------------------------------------------- import copy -import cgi from json import loads from typing import TYPE_CHECKING, cast @@ -34,7 +33,6 @@ from .._utils import _case_insensitive_dict from ._helpers import ( FilesType, - lookup_encoding, parse_lines_from_text, set_content_body, set_json_body, @@ -43,6 +41,7 @@ format_parameters, to_pipeline_transport_request_helper, from_pipeline_transport_request_helper, + get_charset_encoding, ) from ..exceptions import ResponseNotReadError if TYPE_CHECKING: @@ -214,28 +213,16 @@ def url(self): """Returns the URL that resulted in this response""" return self.request.url - def _get_charset_encoding(self): - content_type = self.headers.get("Content-Type") - - if not content_type: - return None - _, params = cgi.parse_header(content_type) - encoding = params.get('charset') # -> utf-8 - if encoding is None or not lookup_encoding(encoding): - return None - return encoding - @property def encoding(self): # type: (...) -> Optional[str] """Returns the response encoding. By default, is specified by the response Content-Type header. """ - try: return self._encoding except AttributeError: - return self._get_charset_encoding() + return get_charset_encoding(self) @encoding.setter def encoding(self, value): diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index 532ab323a811..abe2047c7a64 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -24,7 +24,6 @@ # # -------------------------------------------------------------------------- import copy -import cgi import collections import collections.abc from json import loads @@ -39,6 +38,7 @@ Union, ) + from azure.core.exceptions import HttpResponseError from .._utils import _case_insensitive_dict @@ -48,7 +48,6 @@ FilesType, HeadersType, cast, - lookup_encoding, parse_lines_from_text, set_json_body, set_multipart_body, @@ -56,6 +55,7 @@ format_parameters, to_pipeline_transport_request_helper, from_pipeline_transport_request_helper, + get_charset_encoding ) from ._helpers_py3 import set_content_body from ..exceptions import ResponseNotReadError @@ -242,17 +242,6 @@ def url(self) -> str: """Returns the URL that resulted in this response""" return self.request.url - def _get_charset_encoding(self) -> Optional[str]: - content_type = self.headers.get("Content-Type") - - if not content_type: - return None - _, params = cgi.parse_header(content_type) - encoding = params.get('charset') # -> utf-8 - if encoding is None or not lookup_encoding(encoding): - return None - return encoding - @property def encoding(self) -> Optional[str]: """Returns the response encoding. By default, is specified @@ -261,7 +250,7 @@ def encoding(self) -> Optional[str]: try: return self._encoding except AttributeError: - return self._get_charset_encoding() + return get_charset_encoding(self) @encoding.setter def encoding(self, value: str) -> None: diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py index 61066ee4a35e..317b74c3bac0 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_http_response_async.py @@ -149,7 +149,7 @@ async def test_response_no_charset_with_ascii_content(send_request): assert response.headers["Content-Type"] == "text/plain" assert response.status_code == 200 - assert response.encoding is None + assert response.encoding == 'ascii' content = await response.read() assert content == b"Hello, world!" assert response.text == "Hello, world!" @@ -166,7 +166,7 @@ async def test_response_no_charset_with_iso_8859_1_content(send_request): ) await response.read() assert response.text == u"Accented: Österreich" - assert response.encoding is None + assert response.encoding == 'ISO-8859-1' # NOTE: aiohttp isn't liking this # @pytest.mark.asyncio @@ -187,7 +187,7 @@ async def test_json(send_request): ) await response.read() assert response.json() == {"greeting": "hello", "recipient": "world"} - assert response.encoding is None + assert response.encoding == 'utf-8' @pytest.mark.asyncio async def test_json_with_specified_encoding(send_request): diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py index 804a98b8890e..83255119f4ab 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_http_response.py @@ -138,7 +138,7 @@ def test_response_no_charset_with_ascii_content(send_request): assert response.headers["Content-Type"] == "text/plain" assert response.status_code == 200 - assert response.encoding is None + assert response.encoding == 'ascii' assert response.text == "Hello, world!" @@ -151,7 +151,7 @@ def test_response_no_charset_with_iso_8859_1_content(send_request): request=HttpRequest("GET", "/encoding/iso-8859-1"), ) assert response.text == u"Accented: Österreich" - assert response.encoding is None + assert response.encoding == 'ISO-8859-1' def test_response_set_explicit_encoding(send_request): # Deliberately incorrect charset @@ -168,7 +168,7 @@ def test_json(send_request): request=HttpRequest("GET", "/basic/json"), ) assert response.json() == {"greeting": "hello", "recipient": "world"} - assert response.encoding is None + assert response.encoding == 'utf-8-sig' # for requests, we use utf-8-sig instead of utf-8 bc of requests behavior def test_json_with_specified_encoding(send_request): response = send_request( From 59f0015e119e0896de82688ca94e19121c6eefab Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 20:44:57 -0400 Subject: [PATCH 63/64] read body in a try except so we can close if it fails --- sdk/core/azure-core/azure/core/_pipeline_client.py | 8 ++++++-- .../azure/core/_pipeline_client_async.py | 14 +++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client.py b/sdk/core/azure-core/azure/core/_pipeline_client.py index a17353671c73..ec32e2b20964 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client.py @@ -210,9 +210,13 @@ def send_request(self, request, **kwargs): response = pipeline_response.http_response if rest_request: response = _to_rest_response(response) - if not kwargs.get("stream", False): - response.read() + try: + if not kwargs.get("stream", False): + response.read() + response.close() + except Exception as exc: response.close() + raise exc if return_pipeline_response: pipeline_response.http_response = response pipeline_response.http_request = request diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index 6541b2e240cf..357b3d9b917d 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -184,11 +184,15 @@ async def _make_pipeline_call(self, request, **kwargs): if rest_request: rest_response = _to_rest_response(response) if not kwargs.get("stream"): - # in this case, the pipeline transport response already called .load_body(), so - # the body is loaded. instead of doing response.read(), going to set the body - # to the internal content - rest_response._content = response.body() # pylint: disable=protected-access - await rest_response.close() + try: + # in this case, the pipeline transport response already called .load_body(), so + # the body is loaded. instead of doing response.read(), going to set the body + # to the internal content + rest_response._content = response.body() # pylint: disable=protected-access + await rest_response.close() + except Exception as exc: + await rest_response.close() + raise exc response = rest_response if return_pipeline_response: pipeline_response.http_response = response From a5755497d0ade7880f4e95a4003f28a04160f546 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 28 Jun 2021 21:07:30 -0400 Subject: [PATCH 64/64] remove num_bytes_downloaded --- sdk/core/azure-core/azure/core/rest/_helpers_py3.py | 2 -- sdk/core/azure-core/azure/core/rest/_requests_basic.py | 1 - sdk/core/azure-core/azure/core/rest/_rest.py | 9 --------- sdk/core/azure-core/azure/core/rest/_rest_py3.py | 10 ---------- .../async_tests/test_rest_stream_responses_async.py | 10 ---------- .../testserver_tests/test_rest_stream_responses.py | 9 --------- 6 files changed, 41 deletions(-) diff --git a/sdk/core/azure-core/azure/core/rest/_helpers_py3.py b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py index 4c09b3dbe1ab..90948012db2a 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers_py3.py @@ -87,7 +87,6 @@ async def iter_bytes_helper( stream_download_generator=stream_download_generator, response=response, ): - response._num_bytes_downloaded += len(part) yield part async def iter_raw_helper( @@ -99,5 +98,4 @@ async def iter_raw_helper( stream_download_generator=stream_download_generator, response=response, ): - response._num_bytes_downloaded += len(part) yield part diff --git a/sdk/core/azure-core/azure/core/rest/_requests_basic.py b/sdk/core/azure-core/azure/core/rest/_requests_basic.py index 8bf83ad4e835..e8ef734e1275 100644 --- a/sdk/core/azure-core/azure/core/rest/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/rest/_requests_basic.py @@ -104,7 +104,6 @@ def _stream_download_helper(decompress, response): decompress=decompress, ) for part in stream_download: - response._num_bytes_downloaded += len(part) yield part class RestRequestsTransportResponse(HttpResponse, _RestRequestsTransportResponseBase): diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index e9345eef01b3..24897c6f61dd 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -201,7 +201,6 @@ def __init__(self, **kwargs): self.reason = None self.is_closed = False self.is_stream_consumed = False - self._num_bytes_downloaded = 0 self.content_type = None self._json = None # this is filled in ContentDecodePolicy, when we deserialize self._connection_data_block_size = None # type: Optional[int] @@ -239,12 +238,6 @@ def text(self): encoding = "utf-8-sig" return self.content.decode(encoding) - @property - def num_bytes_downloaded(self): - # type: (...) -> int - """See how many bytes of your stream response have been downloaded""" - return self._num_bytes_downloaded - def json(self): # type: (...) -> Any """Returns the whole body as a json object. @@ -316,8 +309,6 @@ class HttpResponse(_HttpResponseBase): # pylint: disable=too-many-instance-attr :ivar bool is_closed: Whether the network connection has been closed yet :ivar bool is_stream_consumed: When getting a stream response, checks whether the stream has been fully consumed - :ivar int num_bytes_downloaded: The number of bytes in your stream that - have been downloaded """ def __enter__(self): diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index abe2047c7a64..27128c66a94e 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -231,7 +231,6 @@ def __init__( self.reason = None self.is_closed = False self.is_stream_consumed = False - self._num_bytes_downloaded = 0 self.content_type = None self._connection_data_block_size = None self._json = None # this is filled in ContentDecodePolicy, when we deserialize @@ -265,11 +264,6 @@ def text(self) -> str: encoding = "utf-8-sig" return self.content.decode(encoding) - @property - def num_bytes_downloaded(self) -> int: - """See how many bytes of your stream response have been downloaded""" - return self._num_bytes_downloaded - def json(self) -> Any: """Returns the whole body as a json object. @@ -329,8 +323,6 @@ class HttpResponse(_HttpResponseBase): :ivar bool is_closed: Whether the network connection has been closed yet :ivar bool is_stream_consumed: When getting a stream response, checks whether the stream has been fully consumed - :ivar int num_bytes_downloaded: The number of bytes in your stream that - have been downloaded """ def __enter__(self) -> "HttpResponse": @@ -435,8 +427,6 @@ class AsyncHttpResponse(_HttpResponseBase): :ivar bool is_closed: Whether the network connection has been closed yet :ivar bool is_stream_consumed: When getting a stream response, checks whether the stream has been fully consumed - :ivar int num_bytes_downloaded: The number of bytes in your stream that - have been downloaded """ async def read(self) -> bytes: diff --git a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py index b4968ff71cd9..673148749719 100644 --- a/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/async_tests/test_rest_stream_responses_async.py @@ -55,16 +55,6 @@ async def test_iter_with_error(client): raise ValueError("Should error before entering") assert response.is_closed -@pytest.mark.asyncio -async def test_iter_raw_num_bytes_downloaded(client): - request = HttpRequest("GET", "/streams/basic") - - async with client.send_request(request, stream=True) as response: - num_downloaded = response.num_bytes_downloaded - async for part in response.iter_raw(): - assert len(part) == (response.num_bytes_downloaded - num_downloaded) - num_downloaded = response.num_bytes_downloaded - @pytest.mark.asyncio async def test_iter_bytes(client): request = HttpRequest("GET", "/streams/basic") diff --git a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py index 209c12f2fdc6..61053ca7abb9 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_rest_stream_responses.py @@ -64,15 +64,6 @@ def test_iter_with_error(client): raise ValueError("Should error before entering") assert response.is_closed -def test_iter_raw_num_bytes_downloaded(client): - request = HttpRequest("GET", "/streams/basic") - - with client.send_request(request, stream=True) as response: - num_downloaded = response.num_bytes_downloaded - for part in response.iter_raw(): - assert len(part) == (response.num_bytes_downloaded - num_downloaded) - num_downloaded = response.num_bytes_downloaded - def test_iter_bytes(client): request = HttpRequest("GET", "/streams/basic")