From 6dff5d53cff5552c8aa7f4deb75a8f8b225d2a1c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 12:55:33 -0400 Subject: [PATCH 01/28] tests passing locally --- sdk/core/azure-core/dev_requirements.txt | 1 + .../async_tests/test_testserver_async.py | 37 +++++++++++++++++++ sdk/core/azure-core/tests/conftest.py | 25 ++++++++++++- sdk/core/azure-core/tests/pytest.ini | 3 ++ sdk/core/azure-core/tests/test_testserver.py | 34 +++++++++++++++++ 5 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 sdk/core/azure-core/tests/async_tests/test_testserver_async.py create mode 100644 sdk/core/azure-core/tests/pytest.ini create mode 100644 sdk/core/azure-core/tests/test_testserver.py diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index f1cabe1cb127..8269e1064860 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -7,3 +7,4 @@ opencensus-ext-threading mock; python_version < '3.3' -e ../../../tools/azure-sdk-tools -e ../../../tools/azure-devtools +git+https://github.com/iscai-msft/core.testserver#subdirectory=coretestserver \ No newline at end of file diff --git a/sdk/core/azure-core/tests/async_tests/test_testserver_async.py b/sdk/core/azure-core/tests/async_tests/test_testserver_async.py new file mode 100644 index 000000000000..7cc87341678b --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/test_testserver_async.py @@ -0,0 +1,37 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import pytest +from azure.core.pipeline.transport import HttpRequest, AioHttpTransport +"""This file does a simple call to the testserver to make sure we can use the testserver""" + +@pytest.mark.asyncio +async def test_smoke(): + request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") + async with AioHttpTransport() as sender: + response = await sender.send(request) + response.raise_for_status() + await response.load_body() + assert response.text() == "Hello, world!" \ No newline at end of file diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index 0c5e14c094c7..b4f8193c3041 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -23,14 +23,37 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +import pytest +import signal +import os +import subprocess import sys +def start_testserver(): + cmd = "FLASK_APP=coretestserver flask run" + if os.name == 'nt': #On windows, subprocess creation works without being in the shell + return subprocess.Popen(cmd.format("set")) + + return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + +def terminate_testserver(process): + if os.name == 'nt': + process.kill() + else: + os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups + +@pytest.fixture(scope="session") +def testserver(): + """Start the Autorest testserver.""" + server = start_testserver() + yield + # terminate_testserver(server) + # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): collect_ignore.append("async_tests") - # If opencensus is loadable while doing these tests, register an empty tracer to avoid this: # https://github.com/census-instrumentation/opencensus-python/issues/442 try: diff --git a/sdk/core/azure-core/tests/pytest.ini b/sdk/core/azure-core/tests/pytest.ini new file mode 100644 index 000000000000..ab17f621bd0c --- /dev/null +++ b/sdk/core/azure-core/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +usefixtures=testserver +xfail_strict=true \ No newline at end of file diff --git a/sdk/core/azure-core/tests/test_testserver.py b/sdk/core/azure-core/tests/test_testserver.py new file mode 100644 index 000000000000..1a2ba43c9dae --- /dev/null +++ b/sdk/core/azure-core/tests/test_testserver.py @@ -0,0 +1,34 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# -------------------------------------------------------------------------- +from azure.core.pipeline.transport import HttpRequest, RequestsTransport +"""This file does a simple call to the testserver to make sure we can use the testserver""" + +def test_smoke(): + request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") + with RequestsTransport() as sender: + response = sender.send(request) + response.raise_for_status() + assert response.text() == "Hello, world!" From 58c184a8f79098cb828fc2c91eb2d1b14ea364d5 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 14:05:33 -0400 Subject: [PATCH 02/28] move pytests.init to root --- sdk/core/azure-core/{tests => }/pytest.ini | 0 sdk/core/azure-core/samples/conftest.py | 9 +++++++++ 2 files changed, 9 insertions(+) rename sdk/core/azure-core/{tests => }/pytest.ini (100%) diff --git a/sdk/core/azure-core/tests/pytest.ini b/sdk/core/azure-core/pytest.ini similarity index 100% rename from sdk/core/azure-core/tests/pytest.ini rename to sdk/core/azure-core/pytest.ini diff --git a/sdk/core/azure-core/samples/conftest.py b/sdk/core/azure-core/samples/conftest.py index 6d453aed7c4c..0f55930dc4d3 100644 --- a/sdk/core/azure-core/samples/conftest.py +++ b/sdk/core/azure-core/samples/conftest.py @@ -23,8 +23,17 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +import pytest import sys +@pytest.fixture(scope="session") +def testserver(): + # dummy testserver for now + # pytest.ini needs to be in root bc we run pytest on the pipelines with just "pytest" + # because of this, samples conftest needs its own def of testserver. + # plan to change the samples tests to use testserver as well, so it's not always going to be a gross dummy fixture + yield + # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): From c7663fee90eac904d1f9778cfdd3b289619cce26 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 14:58:50 -0400 Subject: [PATCH 03/28] remove set formatting from windows run --- sdk/core/azure-core/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index b4f8193c3041..08aecea27742 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -32,7 +32,7 @@ def start_testserver(): cmd = "FLASK_APP=coretestserver flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - return subprocess.Popen(cmd.format("set")) + return subprocess.Popen(cmd) return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True From bec31ed616f5e36ed34b28cdda86476fbe885cac Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 16:21:08 -0400 Subject: [PATCH 04/28] try to fix windows testserver start --- sdk/core/azure-core/tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index 08aecea27742..f4a614f38385 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -30,11 +30,11 @@ import sys def start_testserver(): - cmd = "FLASK_APP=coretestserver flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - return subprocess.Popen(cmd) + os.environ["FLASK_APP"] = "coretestserver" + return subprocess.Popen("flask run", env=dict(os.environ)) - return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + return subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True def terminate_testserver(process): if os.name == 'nt': From 405d5e83148852e3c5c2d99c2d0377d930a821c1 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 8 Jun 2021 18:08:15 -0400 Subject: [PATCH 05/28] uncomment testserver termination --- sdk/core/azure-core/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index f4a614f38385..f7e7f5cad07f 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -47,7 +47,7 @@ def testserver(): """Start the Autorest testserver.""" server = start_testserver() yield - # terminate_testserver(server) + terminate_testserver(server) # Ignore collection of async tests for Python 2 collect_ignore = [] From f8263774af9ef0f20ecbc3db9d98e4288fe2fe35 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 17 Jun 2021 18:34:56 -0400 Subject: [PATCH 06/28] add devops scripts to start testserver --- scripts/devops_tasks/end_coretestserver.py | 25 ++++++++++++++++++++ scripts/devops_tasks/start_coretestserver.py | 14 +++++++++++ sdk/core/azure-core/tests/conftest.py | 9 ++++--- sdk/core/ci.yml | 12 ++++++++++ 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 scripts/devops_tasks/end_coretestserver.py create mode 100644 scripts/devops_tasks/start_coretestserver.py diff --git a/scripts/devops_tasks/end_coretestserver.py b/scripts/devops_tasks/end_coretestserver.py new file mode 100644 index 000000000000..a900ad160939 --- /dev/null +++ b/scripts/devops_tasks/end_coretestserver.py @@ -0,0 +1,25 @@ +import os +import signal +import argparse + +def end_testserver(pid): + + if os.name == 'nt': + os.kill(pid, signal.CTRL_C_EVENT) + else: + os.killpg(os.getpgid(pid), signal.SIGTERM) # Send the signal to all the process groups + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="Stop the testserver" + ) + parser.add_argument( + "-p", + "--pid", + dest="pid", + help="The pid of the subprocess the testserver is running on", + required=True, + ) + + args = parser.parse_args() + end_testserver(int(args.pid)) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py new file mode 100644 index 000000000000..7301b7c57bb2 --- /dev/null +++ b/scripts/devops_tasks/start_coretestserver.py @@ -0,0 +1,14 @@ +import os +import subprocess + +def start_testserver(): + if os.name == 'nt': #On windows, subprocess creation works without being in the shell + os.environ["FLASK_APP"] = "coretestserver" + result = subprocess.Popen("flask run", env=dict(os.environ)) + else: + result = subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + print('##vso[task.setvariable variable=FLASK_PID]{}'.format(result.pid)) + print("This is used in the pipelines to set the FLASK_PID env var. If you want to stop this testserver, kill this PID.") + +if __name__ == "__main__": + start_testserver() diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index f7e7f5cad07f..aca1030569f4 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -45,9 +45,12 @@ def terminate_testserver(process): @pytest.fixture(scope="session") def testserver(): """Start the Autorest testserver.""" - server = start_testserver() - yield - terminate_testserver(server) + if not os.environ.get("FLASK_PID"): + server = start_testserver() + yield + terminate_testserver(server) + else: + yield # Ignore collection of async tests for Python 2 collect_ignore = [] diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index c569bf00fc25..f53d9afa927e 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -44,6 +44,18 @@ extends: safeName: azurecorecoretracingtelemetry - name: azure-common safeName: azurecommon + BeforeTestSteps: + - task: PythonScript@0 + inputs: + scriptSource: 'scripts/devops_tasks/start_coretestserver.py' + displayName: "Start CoreTestServer" + AfterTestSteps: + - task: PythonScript@0 + inputs: + scriptSource: 'scripts/devops_tasks/end_coretestserver.py' + arguments: >- + -p $(FLASK_PID) + displayName: "Shut down CoreTestServer" CondaArtifacts: - name: azure-core meta_source: conda-recipe/meta.yaml From 6ceb7950aaa22a300ac250b400fc9414be8ca651 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 17 Jun 2021 19:42:30 -0400 Subject: [PATCH 07/28] scriptSource -> scriptPath --- sdk/core/ci.yml | 72 ++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index f53d9afa927e..df0f99a423ec 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -3,64 +3,64 @@ trigger: branches: include: - - master - - main - - hotfix/* - - release/* - - restapi* + - master + - main + - hotfix/* + - release/* + - restapi* paths: include: - - sdk/core/ - - eng/ - - tools/ + - sdk/core/ + - eng/ + - tools/ pr: branches: include: - - master - - main - - feature/* - - hotfix/* - - release/* - - restapi* + - master + - main + - feature/* + - hotfix/* + - release/* + - restapi* paths: include: - - sdk/core/ - - eng/ - - tools/ + - sdk/core/ + - eng/ + - tools/ extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core Artifacts: - - name: azure-core - safeName: azurecore - - name: azure-mgmt-core - safeName: azuremgmtcore - - name: azure-core-tracing-opencensus - safeName: azurecorecoretracingopencensus - - name: azure-core-tracing-opentelemetry - safeName: azurecorecoretracingtelemetry - - name: azure-common - safeName: azurecommon + - name: azure-core + safeName: azurecore + - name: azure-mgmt-core + safeName: azuremgmtcore + - name: azure-core-tracing-opencensus + safeName: azurecorecoretracingopencensus + - name: azure-core-tracing-opentelemetry + safeName: azurecorecoretracingtelemetry + - name: azure-common + safeName: azurecommon BeforeTestSteps: - task: PythonScript@0 inputs: - scriptSource: 'scripts/devops_tasks/start_coretestserver.py' + scriptPath: "scripts/devops_tasks/start_coretestserver.py" displayName: "Start CoreTestServer" AfterTestSteps: - task: PythonScript@0 inputs: - scriptSource: 'scripts/devops_tasks/end_coretestserver.py' + scriptPath: "scripts/devops_tasks/end_coretestserver.py" arguments: >- -p $(FLASK_PID) displayName: "Shut down CoreTestServer" CondaArtifacts: - - name: azure-core - meta_source: conda-recipe/meta.yaml - common_root: azure - checkout: - - package: azure-core - checkout_path: sdk/core - version: 1.12.0 \ No newline at end of file + - name: azure-core + meta_source: conda-recipe/meta.yaml + common_root: azure + checkout: + - package: azure-core + checkout_path: sdk/core + version: 1.12.0 From 0f10e5ea1ac07863b325be897d6c57dfa6d201ee Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 18 Jun 2021 15:08:25 -0400 Subject: [PATCH 08/28] set env vars --- scripts/devops_tasks/start_coretestserver.py | 7 ++++--- sdk/core/azure-core/tests/conftest.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py index 7301b7c57bb2..e9a33760fd47 100644 --- a/scripts/devops_tasks/start_coretestserver.py +++ b/scripts/devops_tasks/start_coretestserver.py @@ -2,11 +2,12 @@ import subprocess def start_testserver(): + os.environ["FLASK_APP"] = "coretestserver" + cmd = "flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - os.environ["FLASK_APP"] = "coretestserver" - result = subprocess.Popen("flask run", env=dict(os.environ)) + result = subprocess.Popen(cmd, env=dict(os.environ)) else: - result = subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + result = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True print('##vso[task.setvariable variable=FLASK_PID]{}'.format(result.pid)) print("This is used in the pipelines to set the FLASK_PID env var. If you want to stop this testserver, kill this PID.") diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index aca1030569f4..66698b53b5a1 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -30,11 +30,12 @@ import sys def start_testserver(): + os.environ["FLASK_APP"] = "coretestserver" + cmd = "flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - os.environ["FLASK_APP"] = "coretestserver" - return subprocess.Popen("flask run", env=dict(os.environ)) + return subprocess.Popen(cmd, env=dict(os.environ)) - return subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True def terminate_testserver(process): if os.name == 'nt': From c35fdc2cb1d62c1792f86bcf95d688733822aa4c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 18 Jun 2021 19:36:08 -0400 Subject: [PATCH 09/28] add pwsh testserver --- sdk/core/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index df0f99a423ec..9af9926587c3 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -45,6 +45,9 @@ extends: - name: azure-common safeName: azurecommon BeforeTestSteps: + - pwsh: | + pip install git+https://github.com/iscai-msft/core.testserver#subdirectory=coretestserver + displayName: "Pip install CoreTestServer" - task: PythonScript@0 inputs: scriptPath: "scripts/devops_tasks/start_coretestserver.py" From f8fd247f39242f0dedfc5e2be8adba8a421651ad Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 11:00:48 -0400 Subject: [PATCH 10/28] return result --- scripts/devops_tasks/start_coretestserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py index e9a33760fd47..d38b6f963d2f 100644 --- a/scripts/devops_tasks/start_coretestserver.py +++ b/scripts/devops_tasks/start_coretestserver.py @@ -10,6 +10,7 @@ def start_testserver(): result = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True print('##vso[task.setvariable variable=FLASK_PID]{}'.format(result.pid)) print("This is used in the pipelines to set the FLASK_PID env var. If you want to stop this testserver, kill this PID.") + return result if __name__ == "__main__": start_testserver() From c4fd6d2858ec9b3748e61fb27e9e470d2b013edf Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 11:38:15 -0400 Subject: [PATCH 11/28] try retuning exit code --- scripts/devops_tasks/start_coretestserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py index d38b6f963d2f..df18e40e3537 100644 --- a/scripts/devops_tasks/start_coretestserver.py +++ b/scripts/devops_tasks/start_coretestserver.py @@ -1,4 +1,5 @@ import os +import sys import subprocess def start_testserver(): @@ -13,4 +14,5 @@ def start_testserver(): return result if __name__ == "__main__": - start_testserver() + result = start_testserver() + sys.exit(0) From b322e283612f72649a045dff32fb77aaa119bc5e Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 15:28:05 -0400 Subject: [PATCH 12/28] remove ci work, make pytest fixture module level --- sdk/core/azure-core/samples/conftest.py | 9 -- sdk/core/azure-core/tests/conftest.py | 29 +------ .../tests/testserver_tests/conftest.py | 56 +++++++++++++ .../{ => tests/testserver_tests}/pytest.ini | 0 .../{ => testserver_tests}/test_testserver.py | 0 .../test_testserver_async.py | 0 sdk/core/ci.yml | 83 ++++++++----------- 7 files changed, 91 insertions(+), 86 deletions(-) create mode 100644 sdk/core/azure-core/tests/testserver_tests/conftest.py rename sdk/core/azure-core/{ => tests/testserver_tests}/pytest.ini (100%) rename sdk/core/azure-core/tests/{ => testserver_tests}/test_testserver.py (100%) rename sdk/core/azure-core/tests/{async_tests => testserver_tests}/test_testserver_async.py (100%) diff --git a/sdk/core/azure-core/samples/conftest.py b/sdk/core/azure-core/samples/conftest.py index 0f55930dc4d3..6d453aed7c4c 100644 --- a/sdk/core/azure-core/samples/conftest.py +++ b/sdk/core/azure-core/samples/conftest.py @@ -23,17 +23,8 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -import pytest import sys -@pytest.fixture(scope="session") -def testserver(): - # dummy testserver for now - # pytest.ini needs to be in root bc we run pytest on the pipelines with just "pytest" - # because of this, samples conftest needs its own def of testserver. - # plan to change the samples tests to use testserver as well, so it's not always going to be a gross dummy fixture - yield - # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): diff --git a/sdk/core/azure-core/tests/conftest.py b/sdk/core/azure-core/tests/conftest.py index 66698b53b5a1..0c5e14c094c7 100644 --- a/sdk/core/azure-core/tests/conftest.py +++ b/sdk/core/azure-core/tests/conftest.py @@ -23,41 +23,14 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -import pytest -import signal -import os -import subprocess import sys -def start_testserver(): - os.environ["FLASK_APP"] = "coretestserver" - cmd = "flask run" - if os.name == 'nt': #On windows, subprocess creation works without being in the shell - return subprocess.Popen(cmd, env=dict(os.environ)) - - return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True - -def terminate_testserver(process): - if os.name == 'nt': - process.kill() - else: - os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups - -@pytest.fixture(scope="session") -def testserver(): - """Start the Autorest testserver.""" - if not os.environ.get("FLASK_PID"): - server = start_testserver() - yield - terminate_testserver(server) - else: - yield - # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): collect_ignore.append("async_tests") + # If opencensus is loadable while doing these tests, register an empty tracer to avoid this: # https://github.com/census-instrumentation/opencensus-python/issues/442 try: diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py new file mode 100644 index 000000000000..beb636073c11 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -0,0 +1,56 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- +import pytest +import signal +import os +import subprocess +import sys + +def start_testserver(): + os.environ["FLASK_APP"] = "coretestserver" + cmd = "flask run" + if os.name == 'nt': #On windows, subprocess creation works without being in the shell + return subprocess.Popen(cmd, env=dict(os.environ)) + + return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + +def terminate_testserver(process): + if os.name == 'nt': + process.kill() + else: + os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups + +@pytest.fixture(scope="module") +def testserver(): + """Start the Autorest testserver.""" + server = start_testserver() + yield + terminate_testserver(server) + +# Ignore collection of async tests for Python 2 +collect_ignore = [] +if sys.version_info < (3, 5): + collect_ignore.append("*_async.py") diff --git a/sdk/core/azure-core/pytest.ini b/sdk/core/azure-core/tests/testserver_tests/pytest.ini similarity index 100% rename from sdk/core/azure-core/pytest.ini rename to sdk/core/azure-core/tests/testserver_tests/pytest.ini diff --git a/sdk/core/azure-core/tests/test_testserver.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py similarity index 100% rename from sdk/core/azure-core/tests/test_testserver.py rename to sdk/core/azure-core/tests/testserver_tests/test_testserver.py diff --git a/sdk/core/azure-core/tests/async_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py similarity index 100% rename from sdk/core/azure-core/tests/async_tests/test_testserver_async.py rename to sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 9af9926587c3..c569bf00fc25 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -3,67 +3,52 @@ trigger: branches: include: - - master - - main - - hotfix/* - - release/* - - restapi* + - master + - main + - hotfix/* + - release/* + - restapi* paths: include: - - sdk/core/ - - eng/ - - tools/ + - sdk/core/ + - eng/ + - tools/ pr: branches: include: - - master - - main - - feature/* - - hotfix/* - - release/* - - restapi* + - master + - main + - feature/* + - hotfix/* + - release/* + - restapi* paths: include: - - sdk/core/ - - eng/ - - tools/ + - sdk/core/ + - eng/ + - tools/ extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core Artifacts: - - name: azure-core - safeName: azurecore - - name: azure-mgmt-core - safeName: azuremgmtcore - - name: azure-core-tracing-opencensus - safeName: azurecorecoretracingopencensus - - name: azure-core-tracing-opentelemetry - safeName: azurecorecoretracingtelemetry - - name: azure-common - safeName: azurecommon - BeforeTestSteps: - - pwsh: | - pip install git+https://github.com/iscai-msft/core.testserver#subdirectory=coretestserver - displayName: "Pip install CoreTestServer" - - task: PythonScript@0 - inputs: - scriptPath: "scripts/devops_tasks/start_coretestserver.py" - displayName: "Start CoreTestServer" - AfterTestSteps: - - task: PythonScript@0 - inputs: - scriptPath: "scripts/devops_tasks/end_coretestserver.py" - arguments: >- - -p $(FLASK_PID) - displayName: "Shut down CoreTestServer" + - name: azure-core + safeName: azurecore + - name: azure-mgmt-core + safeName: azuremgmtcore + - name: azure-core-tracing-opencensus + safeName: azurecorecoretracingopencensus + - name: azure-core-tracing-opentelemetry + safeName: azurecorecoretracingtelemetry + - name: azure-common + safeName: azurecommon CondaArtifacts: - - name: azure-core - meta_source: conda-recipe/meta.yaml - common_root: azure - checkout: - - package: azure-core - checkout_path: sdk/core - version: 1.12.0 + - name: azure-core + meta_source: conda-recipe/meta.yaml + common_root: azure + checkout: + - package: azure-core + checkout_path: sdk/core + version: 1.12.0 \ No newline at end of file From 449f42a2e99d0fec277159688ac9ee890b75ea89 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 17:32:39 -0400 Subject: [PATCH 13/28] tests working without pytest.ini --- .../azure-core/tests/testserver_tests/conftest.py | 15 +++++++++++---- .../azure-core/tests/testserver_tests/pytest.ini | 3 --- .../tests/testserver_tests/test_testserver.py | 2 ++ .../testserver_tests/test_testserver_async.py | 1 + 4 files changed, 14 insertions(+), 7 deletions(-) delete mode 100644 sdk/core/azure-core/tests/testserver_tests/pytest.ini diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index beb636073c11..cdfea8da4fb5 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -23,6 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +import time import pytest import signal import os @@ -33,9 +34,13 @@ def start_testserver(): os.environ["FLASK_APP"] = "coretestserver" cmd = "flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell - return subprocess.Popen(cmd, env=dict(os.environ)) - - return subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True + child_process = subprocess.Popen(cmd, env=dict(os.environ)) + else: + child_process = subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) + time.sleep(1) + if child_process.returncode is not None: + raise ValueError("Didn't start!") + return child_process #On linux, have to set shell=True def terminate_testserver(process): if os.name == 'nt': @@ -43,13 +48,15 @@ def terminate_testserver(process): else: os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups -@pytest.fixture(scope="module") +@pytest.fixture() def testserver(): """Start the Autorest testserver.""" server = start_testserver() yield terminate_testserver(server) + + # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): diff --git a/sdk/core/azure-core/tests/testserver_tests/pytest.ini b/sdk/core/azure-core/tests/testserver_tests/pytest.ini deleted file mode 100644 index ab17f621bd0c..000000000000 --- a/sdk/core/azure-core/tests/testserver_tests/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -usefixtures=testserver -xfail_strict=true \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py index 1a2ba43c9dae..8cbad018c60d 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py @@ -23,8 +23,10 @@ # THE SOFTWARE. # # -------------------------------------------------------------------------- +import pytest from azure.core.pipeline.transport import HttpRequest, RequestsTransport """This file does a simple call to the testserver to make sure we can use the testserver""" +pytestmark = pytest.mark.usefixtures("testserver") def test_smoke(): request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py index 7cc87341678b..ccb5d6421b4e 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py @@ -26,6 +26,7 @@ import pytest from azure.core.pipeline.transport import HttpRequest, AioHttpTransport """This file does a simple call to the testserver to make sure we can use the testserver""" +pytestmark = pytest.mark.usefixtures("testserver") @pytest.mark.asyncio async def test_smoke(): From a021de277aeb01bd572a07a43863b460ea893dc3 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 17:36:36 -0400 Subject: [PATCH 14/28] only have testserver fixture in conftest --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 3 +-- sdk/core/azure-core/tests/testserver_tests/test_testserver.py | 2 -- .../azure-core/tests/testserver_tests/test_testserver_async.py | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index cdfea8da4fb5..237ba261c166 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -48,7 +48,7 @@ def terminate_testserver(process): else: os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups -@pytest.fixture() +@pytest.fixture(autouse=True, scope="module") def testserver(): """Start the Autorest testserver.""" server = start_testserver() @@ -56,7 +56,6 @@ def testserver(): terminate_testserver(server) - # Ignore collection of async tests for Python 2 collect_ignore = [] if sys.version_info < (3, 5): diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py index 8cbad018c60d..07bbe54c911f 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py @@ -26,8 +26,6 @@ import pytest from azure.core.pipeline.transport import HttpRequest, RequestsTransport """This file does a simple call to the testserver to make sure we can use the testserver""" -pytestmark = pytest.mark.usefixtures("testserver") - def test_smoke(): request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") with RequestsTransport() as sender: diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py index ccb5d6421b4e..7cc87341678b 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py @@ -26,7 +26,6 @@ import pytest from azure.core.pipeline.transport import HttpRequest, AioHttpTransport """This file does a simple call to the testserver to make sure we can use the testserver""" -pytestmark = pytest.mark.usefixtures("testserver") @pytest.mark.asyncio async def test_smoke(): From 7a68dd3d63f8368808f14c1bf99fa84a444f1e47 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 17:45:12 -0400 Subject: [PATCH 15/28] switch to package scope --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 237ba261c166..0827d0c68d3a 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -48,7 +48,7 @@ def terminate_testserver(process): else: os.killpg(os.getpgid(process.pid), signal.SIGTERM) # Send the signal to all the process groups -@pytest.fixture(autouse=True, scope="module") +@pytest.fixture(autouse=True, scope="package") def testserver(): """Start the Autorest testserver.""" server = start_testserver() From df42f157c29e746b0ade8e469c826ae1569e61d7 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 21 Jun 2021 17:53:53 -0400 Subject: [PATCH 16/28] unite testserver setting --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 0827d0c68d3a..12bcda2c02e7 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -31,12 +31,11 @@ import sys def start_testserver(): - os.environ["FLASK_APP"] = "coretestserver" - cmd = "flask run" + cmd = "FLASK_APP=coretestserver flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell child_process = subprocess.Popen(cmd, env=dict(os.environ)) else: - child_process = subprocess.Popen("FLASK_APP=coretestserver flask run", shell=True, preexec_fn=os.setsid) + child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) time.sleep(1) if child_process.returncode is not None: raise ValueError("Didn't start!") From da62f4b8413cb7569118ca9dc2fececdedd897e6 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 11:08:44 -0400 Subject: [PATCH 17/28] switch to environment variables --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 12bcda2c02e7..6c7258d13ba5 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -31,15 +31,17 @@ import sys def start_testserver(): - cmd = "FLASK_APP=coretestserver flask run" + os.environ["FLASK_APP"] = "coretestserver" + cmd = "flask run" if os.name == 'nt': #On windows, subprocess creation works without being in the shell child_process = subprocess.Popen(cmd, env=dict(os.environ)) else: - child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) + #On linux, have to set shell=True + child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid, env=dict(os.environ)) time.sleep(1) if child_process.returncode is not None: raise ValueError("Didn't start!") - return child_process #On linux, have to set shell=True + return child_process def terminate_testserver(process): if os.name == 'nt': From e06ebb4bc553f9c251252a73dd114863e6a380c9 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 15:47:41 -0400 Subject: [PATCH 18/28] see what happens if we don't kill testserver --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 6c7258d13ba5..ecfb5f8c6c5a 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -54,7 +54,6 @@ def testserver(): """Start the Autorest testserver.""" server = start_testserver() yield - terminate_testserver(server) # Ignore collection of async tests for Python 2 From 04ddbd0830a973ea7b66a8f3f92a17092a37455e Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 17:08:36 -0400 Subject: [PATCH 19/28] cycle through ports --- .../tests/testserver_tests/conftest.py | 29 ++++++++++++++++++- .../tests/testserver_tests/test_testserver.py | 5 ++-- .../testserver_tests/test_testserver_async.py | 4 +-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index ecfb5f8c6c5a..86916f40e2da 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -23,16 +23,42 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +from re import T import time import pytest import signal import os import subprocess import sys +import random +import http.client +import urllib + +def is_port_open(port_num): + conn = http.client.HTTPSConnection("localhost:{}".format(port_num)) + try: + conn.request("GET", "/health") + return False + except ConnectionRefusedError: + return True + +def get_port(): + count = 3 + for _ in range(count): + port_num = random.randrange(3000, 5000) + if is_port_open(port_num): + return port_num + raise TypeError("Tried {} times, can't find an open port".format(count)) + +@pytest.fixture +def port(): + return os.environ["FLASK_PORT"] def start_testserver(): + port = get_port() os.environ["FLASK_APP"] = "coretestserver" - cmd = "flask run" + os.environ["FLASK_PORT"] = str(port) + cmd = "flask run -p {}".format(port) if os.name == 'nt': #On windows, subprocess creation works without being in the shell child_process = subprocess.Popen(cmd, env=dict(os.environ)) else: @@ -54,6 +80,7 @@ def testserver(): """Start the Autorest testserver.""" server = start_testserver() yield + terminate_testserver(server) # Ignore collection of async tests for Python 2 diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py index 07bbe54c911f..a31d449fbd53 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver.py @@ -26,8 +26,9 @@ import pytest from azure.core.pipeline.transport import HttpRequest, RequestsTransport """This file does a simple call to the testserver to make sure we can use the testserver""" -def test_smoke(): - request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") + +def test_smoke(port): + request = HttpRequest(method="GET", url="http://localhost:{}/basic/string".format(port)) with RequestsTransport() as sender: response = sender.send(request) response.raise_for_status() diff --git a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py index 7cc87341678b..623033080bd1 100644 --- a/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py +++ b/sdk/core/azure-core/tests/testserver_tests/test_testserver_async.py @@ -28,8 +28,8 @@ """This file does a simple call to the testserver to make sure we can use the testserver""" @pytest.mark.asyncio -async def test_smoke(): - request = HttpRequest(method="GET", url="http://localhost:5000/basic/string") +async def test_smoke(port): + request = HttpRequest(method="GET", url="http://localhost:{}/basic/string".format(port)) async with AioHttpTransport() as sender: response = await sender.send(request) response.raise_for_status() From e860e8aa2caa72c25cd98dd201cf2811dcef2a3c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 17:12:17 -0400 Subject: [PATCH 20/28] remove scripts --- scripts/devops_tasks/end_coretestserver.py | 25 -------------------- scripts/devops_tasks/start_coretestserver.py | 18 -------------- 2 files changed, 43 deletions(-) delete mode 100644 scripts/devops_tasks/end_coretestserver.py delete mode 100644 scripts/devops_tasks/start_coretestserver.py diff --git a/scripts/devops_tasks/end_coretestserver.py b/scripts/devops_tasks/end_coretestserver.py deleted file mode 100644 index a900ad160939..000000000000 --- a/scripts/devops_tasks/end_coretestserver.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import signal -import argparse - -def end_testserver(pid): - - if os.name == 'nt': - os.kill(pid, signal.CTRL_C_EVENT) - else: - os.killpg(os.getpgid(pid), signal.SIGTERM) # Send the signal to all the process groups - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="Stop the testserver" - ) - parser.add_argument( - "-p", - "--pid", - dest="pid", - help="The pid of the subprocess the testserver is running on", - required=True, - ) - - args = parser.parse_args() - end_testserver(int(args.pid)) diff --git a/scripts/devops_tasks/start_coretestserver.py b/scripts/devops_tasks/start_coretestserver.py deleted file mode 100644 index df18e40e3537..000000000000 --- a/scripts/devops_tasks/start_coretestserver.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import sys -import subprocess - -def start_testserver(): - os.environ["FLASK_APP"] = "coretestserver" - cmd = "flask run" - if os.name == 'nt': #On windows, subprocess creation works without being in the shell - result = subprocess.Popen(cmd, env=dict(os.environ)) - else: - result = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) #On linux, have to set shell=True - print('##vso[task.setvariable variable=FLASK_PID]{}'.format(result.pid)) - print("This is used in the pipelines to set the FLASK_PID env var. If you want to stop this testserver, kill this PID.") - return result - -if __name__ == "__main__": - result = start_testserver() - sys.exit(0) From 2f39ad2292122f5d677a5e13e5b5dfed763e7888 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 19:17:58 -0400 Subject: [PATCH 21/28] allow 2.7 compatibility --- .../azure-core/tests/testserver_tests/conftest.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 86916f40e2da..c2dec634874f 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -31,15 +31,17 @@ import subprocess import sys import random -import http.client -import urllib +try: + import http.client as httpclient +except ImportError: + import httplib as httpclient def is_port_open(port_num): - conn = http.client.HTTPSConnection("localhost:{}".format(port_num)) + conn = httpclient.HTTPSConnection("localhost:{}".format(port_num)) try: conn.request("GET", "/health") return False - except ConnectionRefusedError: + except Exception: return True def get_port(): @@ -84,6 +86,6 @@ def testserver(): # Ignore collection of async tests for Python 2 -collect_ignore = [] +collect_ignore_glob = [] if sys.version_info < (3, 5): - collect_ignore.append("*_async.py") + collect_ignore_glob.append("*_async.py") From f503b2cdaf5698786970694941b79beee8f448ec Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 22 Jun 2021 21:41:04 -0400 Subject: [PATCH 22/28] wait longer for pypy --- sdk/core/azure-core/tests/testserver_tests/conftest.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index c2dec634874f..1c7bfe590c54 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -66,10 +66,12 @@ def start_testserver(): else: #On linux, have to set shell=True child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid, env=dict(os.environ)) - time.sleep(1) - if child_process.returncode is not None: - raise ValueError("Didn't start!") - return child_process + count = 5 + for _ in range(count): + time.sleep(1) + if not is_port_open(port): + return child_process + raise ValueError("Didn't start!") def terminate_testserver(process): if os.name == 'nt': From 8ccce3407ce06960e6ff2e3556a6f30878c89472 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 23 Jun 2021 11:45:51 -0400 Subject: [PATCH 23/28] increase sleep to 2 for pypy --- .../azure-core/tests/testserver_tests/conftest.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 1c7bfe590c54..8ddf8a27ab03 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -23,7 +23,6 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from re import T import time import pytest import signal @@ -41,7 +40,7 @@ def is_port_open(port_num): try: conn.request("GET", "/health") return False - except Exception: + except Exception as e: return True def get_port(): @@ -66,12 +65,10 @@ def start_testserver(): else: #On linux, have to set shell=True child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid, env=dict(os.environ)) - count = 5 - for _ in range(count): - time.sleep(1) - if not is_port_open(port): - return child_process - raise ValueError("Didn't start!") + time.sleep(2) + if not child_process.returncode is None: + raise ValueError("Didn't start!") + return child_process def terminate_testserver(process): if os.name == 'nt': From 30c6e39ba424eab3fa4f637826dedc30b2b16ede Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 13:07:04 -0400 Subject: [PATCH 24/28] move core testserver into tests --- sdk/core/azure-core/dev_requirements.txt | 2 +- .../coretestserver/coretestserver/__init__.py | 33 +++++++ .../coretestserver/test_routes/__init__.py | 24 +++++ .../coretestserver/test_routes/basic.py | 66 +++++++++++++ .../coretestserver/test_routes/encoding.py | 92 +++++++++++++++++++ .../coretestserver/test_routes/errors.py | 28 ++++++ .../coretestserver/test_routes/helpers.py | 12 +++ .../coretestserver/test_routes/multipart.py | 88 ++++++++++++++++++ .../coretestserver/test_routes/streams.py | 38 ++++++++ .../coretestserver/test_routes/urlencoded.py | 26 ++++++ .../coretestserver/test_routes/xml_route.py | 46 ++++++++++ .../testserver_tests/coretestserver/setup.py | 35 +++++++ 12 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index 8269e1064860..27a3bb04ce64 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -7,4 +7,4 @@ opencensus-ext-threading mock; python_version < '3.3' -e ../../../tools/azure-sdk-tools -e ../../../tools/azure-devtools -git+https://github.com/iscai-msft/core.testserver#subdirectory=coretestserver \ No newline at end of file +-e tests/testserver_tests/coretestserver \ No newline at end of file diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py new file mode 100644 index 000000000000..63560847a01f --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/__init__.py @@ -0,0 +1,33 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +from flask import Flask, Response +from .test_routes import ( + basic_api, + encoding_api, + errors_api, + streams_api, + urlencoded_api, + multipart_api, + xml_api +) + +app = Flask(__name__) +app.register_blueprint(basic_api, url_prefix="/basic") +app.register_blueprint(encoding_api, url_prefix="/encoding") +app.register_blueprint(errors_api, url_prefix="/errors") +app.register_blueprint(streams_api, url_prefix="/streams") +app.register_blueprint(urlencoded_api, url_prefix="/urlencoded") +app.register_blueprint(multipart_api, url_prefix="/multipart") +app.register_blueprint(xml_api, url_prefix="/xml") + +@app.route('/health', methods=['GET']) +def latin_1_charset_utf8(): + return Response(status=200) + +if __name__ == "__main__": + app.run(debug=True) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py new file mode 100644 index 000000000000..82f4e7ac4566 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/__init__.py @@ -0,0 +1,24 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +from .basic import basic_api +from .encoding import encoding_api +from .errors import errors_api +from .multipart import multipart_api +from .streams import streams_api +from .urlencoded import urlencoded_api +from .xml_route import xml_api + +__all__ = [ + "basic_api", + "encoding_api", + "errors_api", + "multipart_api", + "streams_api", + "urlencoded_api", + "xml_api", +] diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py new file mode 100644 index 000000000000..4b7d5ae92ad4 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/basic.py @@ -0,0 +1,66 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +from flask import ( + Response, + Blueprint, + request +) + +basic_api = Blueprint('basic_api', __name__) + +@basic_api.route('/string', methods=['GET']) +def string(): + return Response( + "Hello, world!", status=200, mimetype="text/plain" + ) + +@basic_api.route('/lines', methods=['GET']) +def lines(): + return Response( + "Hello,\nworld!", status=200, mimetype="text/plain" + ) + +@basic_api.route("/bytes", methods=['GET']) +def bytes(): + return Response( + "Hello, world!".encode(), status=200, mimetype="text/plain" + ) + +@basic_api.route("/html", methods=['GET']) +def html(): + return Response( + "Hello, world!", status=200, mimetype="text/html" + ) + +@basic_api.route("/json", methods=['GET']) +def json(): + return Response( + '{"greeting": "hello", "recipient": "world"}', status=200, mimetype="application/json" + ) + +@basic_api.route("/complicated-json", methods=['POST']) +def complicated_json(): + # thanks to Sean Kane for this test! + assert request.json['EmptyByte'] == '' + assert request.json['EmptyUnicode'] == '' + assert request.json['SpacesOnlyByte'] == ' ' + assert request.json['SpacesOnlyUnicode'] == ' ' + assert request.json['SpacesBeforeByte'] == ' Text' + assert request.json['SpacesBeforeUnicode'] == ' Text' + assert request.json['SpacesAfterByte'] == 'Text ' + assert request.json['SpacesAfterUnicode'] == 'Text ' + assert request.json['SpacesBeforeAndAfterByte'] == ' Text ' + assert request.json['SpacesBeforeAndAfterUnicode'] == ' Text ' + assert request.json['啊齄丂狛'] == 'ꀕ' + assert request.json['RowKey'] == 'test2' + assert request.json['啊齄丂狛狜'] == 'hello' + assert request.json["singlequote"] == "a''''b" + assert request.json["doublequote"] == 'a""""b' + assert request.json["None"] == None + + return Response(status=200) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py new file mode 100644 index 000000000000..12224e568ee5 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/encoding.py @@ -0,0 +1,92 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from json import dumps +from flask import ( + Response, + Blueprint, +) + +encoding_api = Blueprint('encoding_api', __name__) + +@encoding_api.route('/latin-1', methods=['GET']) +def latin_1(): + r = Response( + "Latin 1: ÿ".encode("latin-1"), status=200 + ) + r.headers["Content-Type"] = "text/plain; charset=latin-1" + return r + +@encoding_api.route('/latin-1-with-utf-8', methods=['GET']) +def latin_1_charset_utf8(): + r = Response( + "Latin 1: ÿ".encode("latin-1"), status=200 + ) + r.headers["Content-Type"] = "text/plain; charset=utf-8" + return r + +@encoding_api.route('/no-charset', methods=['GET']) +def latin_1_no_charset(): + r = Response( + "Hello, world!", status=200 + ) + r.headers["Content-Type"] = "text/plain" + return r + +@encoding_api.route('/iso-8859-1', methods=['GET']) +def iso_8859_1(): + r = Response( + "Accented: Österreich".encode("iso-8859-1"), status=200 + ) + r.headers["Content-Type"] = "text/plain" + return r + +@encoding_api.route('/emoji', methods=['GET']) +def emoji(): + r = Response( + "👩", status=200 + ) + return r + +@encoding_api.route('/emoji-family-skin-tone-modifier', methods=['GET']) +def emoji_family_skin_tone_modifier(): + r = Response( + "👩🏻‍👩🏽‍👧🏾‍👦🏿 SSN: 859-98-0987", status=200 + ) + return r + +@encoding_api.route('/korean', methods=['GET']) +def korean(): + r = Response( + "아가", status=200 + ) + return r + +@encoding_api.route('/json', methods=['GET']) +def json(): + data = {"greeting": "hello", "recipient": "world"} + content = dumps(data).encode("utf-16") + r = Response( + content, status=200 + ) + r.headers["Content-Type"] = "application/json; charset=utf-16" + return r + +@encoding_api.route('/invalid-codec-name', methods=['GET']) +def invalid_codec_name(): + r = Response( + "おはようございます。".encode("utf-8"), status=200 + ) + r.headers["Content-Type"] = "text/plain; charset=invalid-codec-name" + return r + +@encoding_api.route('/no-charset', methods=['GET']) +def no_charset(): + r = Response( + "Hello, world!", status=200 + ) + r.headers["Content-Type"] = "text/plain" + return r diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py new file mode 100644 index 000000000000..221f598e063a --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py @@ -0,0 +1,28 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from flask import ( + Response, + Blueprint, +) + +errors_api = Blueprint('errors_api', __name__) + +@errors_api.route('/403', methods=['GET']) +def get_403(): + return Response(status=403) + +@errors_api.route('/500', methods=['GET']) +def get_500(): + return Response(status=500) + +@errors_api.route('/stream', methods=['GET']) +def get_stream(): + class StreamingBody: + def __iter__(self): + yield b"Hello, " + yield b"world!" + return Response(StreamingBody(), status=500) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py new file mode 100644 index 000000000000..46680f65d3f9 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/helpers.py @@ -0,0 +1,12 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- + +def assert_with_message(param_name, expected_value, actual_value): + assert expected_value == actual_value, "Expected '{}' to be '{}', got '{}'".format( + param_name, expected_value, actual_value + ) + +__all__ = ["assert_with_message"] diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py new file mode 100644 index 000000000000..236496673a2f --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/multipart.py @@ -0,0 +1,88 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from copy import copy +from flask import ( + Response, + Blueprint, + request, +) +from .helpers import assert_with_message + +multipart_api = Blueprint('multipart_api', __name__) + +multipart_header_start = "multipart/form-data; boundary=" + +# NOTE: the flask behavior is different for aiohttp and requests +# in requests, we see the file content through request.form +# in aiohttp, we see the file through request.files + +@multipart_api.route('/basic', methods=['POST']) +def basic(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + if request.files: + # aiohttp + assert_with_message("content length", 258, request.content_length) + assert_with_message("num files", 1, len(request.files)) + assert_with_message("has file named fileContent", True, bool(request.files.get('fileContent'))) + file_content = request.files['fileContent'] + assert_with_message("file content type", "application/octet-stream", file_content.content_type) + assert_with_message("file content length", 14, file_content.content_length) + assert_with_message("filename", "fileContent", file_content.filename) + assert_with_message("has content disposition header", True, bool(file_content.headers.get("Content-Disposition"))) + assert_with_message( + "content disposition", + 'form-data; name="fileContent"; filename="fileContent"; filename*=utf-8\'\'fileContent', + file_content.headers["Content-Disposition"] + ) + elif request.form: + # requests + assert_with_message("content length", 184, request.content_length) + assert_with_message("fileContent", "", request.form["fileContent"]) + else: + return Response(status=400) # should be either of these + return Response(status=200) + +@multipart_api.route('/data-and-files', methods=['POST']) +def data_and_files(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + assert_with_message("message", "Hello, world!", request.form["message"]) + assert_with_message("message", "", request.form["fileContent"]) + return Response(status=200) + +@multipart_api.route('/data-and-files-tuple', methods=['POST']) +def data_and_files_tuple(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + assert_with_message("message", ["abc"], request.form["message"]) + assert_with_message("message", [""], request.form["fileContent"]) + return Response(status=200) + +@multipart_api.route('/non-seekable-filelike', methods=['POST']) +def non_seekable_filelike(): + assert_with_message("content type", multipart_header_start, request.content_type[:len(multipart_header_start)]) + if request.files: + # aiohttp + len_files = len(request.files) + assert_with_message("num files", 1, len_files) + # assert_with_message("content length", 258, request.content_length) + assert_with_message("num files", 1, len(request.files)) + assert_with_message("has file named file", True, bool(request.files.get('file'))) + file = request.files['file'] + assert_with_message("file content type", "application/octet-stream", file.content_type) + assert_with_message("file content length", 14, file.content_length) + assert_with_message("filename", "file", file.filename) + assert_with_message("has content disposition header", True, bool(file.headers.get("Content-Disposition"))) + assert_with_message( + "content disposition", + 'form-data; name="fileContent"; filename="fileContent"; filename*=utf-8\'\'fileContent', + file.headers["Content-Disposition"] + ) + elif request.form: + # requests + assert_with_message("num files", 1, len(request.form)) + else: + return Response(status=400) + return Response(status=200) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py new file mode 100644 index 000000000000..1aeb7c05cc21 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/streams.py @@ -0,0 +1,38 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from flask import ( + Response, + Blueprint, +) + +streams_api = Blueprint('streams_api', __name__) + +class StreamingBody: + def __iter__(self): + yield b"Hello, " + yield b"world!" + + +def streaming_body(): + yield b"Hello, " + yield b"world!" + +def stream_json_error(): + yield '{"error": {"code": "BadRequest", ' + yield' "message": "You made a bad request"}}' + +@streams_api.route('/basic', methods=['GET']) +def basic(): + return Response(streaming_body(), status=200) + +@streams_api.route('/iterable', methods=['GET']) +def iterable(): + return Response(StreamingBody(), status=200) + +@streams_api.route('/error', methods=['GET']) +def error(): + return Response(stream_json_error(), status=400) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py new file mode 100644 index 000000000000..4ea2bdd2795d --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/urlencoded.py @@ -0,0 +1,26 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from flask import ( + Response, + Blueprint, + request, +) +from .helpers import assert_with_message + +urlencoded_api = Blueprint('urlencoded_api', __name__) + +@urlencoded_api.route('/pet/add/', methods=['POST']) +def basic(pet_id): + assert_with_message("pet_id", "1", pet_id) + assert_with_message("content type", "application/x-www-form-urlencoded", request.content_type) + assert_with_message("content length", 47, request.content_length) + assert len(request.form) == 4 + assert_with_message("pet_type", "dog", request.form["pet_type"]) + assert_with_message("pet_food", "meat", request.form["pet_food"]) + assert_with_message("name", "Fido", request.form["name"]) + assert_with_message("pet_age", "42", request.form["pet_age"]) + return Response(status=200) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py new file mode 100644 index 000000000000..c19aed97b6b5 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/xml_route.py @@ -0,0 +1,46 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +import xml.etree.ElementTree as ET +from flask import ( + Response, + Blueprint, + request, +) +from .helpers import assert_with_message + +xml_api = Blueprint('xml_api', __name__) + +@xml_api.route('/basic', methods=['GET', 'PUT']) +def basic(): + basic_body = """ + + + Wake up to WonderWidgets! + + + Overview + Why WonderWidgets are great + + Who buys WonderWidgets + +""" + + if request.method == 'GET': + return Response(basic_body, status=200) + elif request.method == 'PUT': + assert_with_message("content length", str(len(request.data)), request.headers["Content-Length"]) + parsed_xml = ET.fromstring(request.data.decode("utf-8")) + assert_with_message("tag", "slideshow", parsed_xml.tag) + attributes = parsed_xml.attrib + assert_with_message("title attribute", "Sample Slide Show", attributes['title']) + assert_with_message("date attribute", "Date of publication", attributes['date']) + assert_with_message("author attribute", "Yours Truly", attributes['author']) + return Response(status=200) + return Response("You have passed in method '{}' that is not 'GET' or 'PUT'".format(request.method), status=400) diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py new file mode 100644 index 000000000000..a43288221498 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/setup.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from setuptools import setup, find_packages + +version = "1.0.0b1" + +setup( + name="coretestserver", + version=version, + include_package_data=True, + description='Testserver for Python Core', + long_description='Testserver for Python Core', + license='MIT License', + author='Microsoft Corporation', + author_email='azpysdkhelp@microsoft.com', + url='https://github.com/iscai-msft/core.testserver', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'License :: OSI Approved :: MIT License', + ], + packages=find_packages(), + install_requires=[ + "flask" + ] +) From 20e593a8e28f53653d49a6b19510aa51c57002ad Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 13:29:05 -0400 Subject: [PATCH 25/28] switch to urllib requesting to see if port open --- .../tests/testserver_tests/conftest.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/sdk/core/azure-core/tests/testserver_tests/conftest.py b/sdk/core/azure-core/tests/testserver_tests/conftest.py index 8ddf8a27ab03..10a99fb3ce21 100644 --- a/sdk/core/azure-core/tests/testserver_tests/conftest.py +++ b/sdk/core/azure-core/tests/testserver_tests/conftest.py @@ -30,16 +30,12 @@ import subprocess import sys import random -try: - import http.client as httpclient -except ImportError: - import httplib as httpclient +from six.moves import urllib -def is_port_open(port_num): - conn = httpclient.HTTPSConnection("localhost:{}".format(port_num)) +def is_port_available(port_num): + req = urllib.request.Request("http://localhost:{}/health".format(port_num)) try: - conn.request("GET", "/health") - return False + return urllib.request.urlopen(req).code != 200 except Exception as e: return True @@ -47,7 +43,7 @@ def get_port(): count = 3 for _ in range(count): port_num = random.randrange(3000, 5000) - if is_port_open(port_num): + if is_port_available(port_num): return port_num raise TypeError("Tried {} times, can't find an open port".format(count)) @@ -65,10 +61,12 @@ def start_testserver(): else: #On linux, have to set shell=True child_process = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid, env=dict(os.environ)) - time.sleep(2) - if not child_process.returncode is None: - raise ValueError("Didn't start!") - return child_process + count = 5 + for _ in range(count): + if not is_port_available(port): + return child_process + time.sleep(1) + raise ValueError("Didn't start!") def terminate_testserver(process): if os.name == 'nt': From 1483f8cd7765f713f468069e205ba2b86f7415cb Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 24 Jun 2021 13:57:01 -0400 Subject: [PATCH 26/28] ignore coretestserver readme --- eng/.docsettings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml index 9b2268542ee7..940dcaddd1bc 100644 --- a/eng/.docsettings.yml +++ b/eng/.docsettings.yml @@ -93,6 +93,7 @@ known_content_issues: - ['sdk/containerregistry/azure-containerregistry/swagger/README.md', '#4554'] - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554'] - ['sdk/attestation/azure-security-attestation/swagger/README.md', '#4554'] + - ['sdk/core/azure-core/tests/testserver_tests/coretestserver/README.md', '#4554'] # common. - ['sdk/appconfiguration/azure-appconfiguration/README.md', 'common'] From 3d1273534e31249f5dafd589bef4db60a359ad47 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 11:47:28 -0400 Subject: [PATCH 27/28] add readme rst --- .../coretestserver/README.rst | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 sdk/core/azure-core/tests/testserver_tests/coretestserver/README.rst diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/README.rst b/sdk/core/azure-core/tests/testserver_tests/coretestserver/README.rst new file mode 100644 index 000000000000..9ac3fbf88847 --- /dev/null +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/README.rst @@ -0,0 +1,22 @@ +Testserver for Python Azure Core +============================================== + +This package contains a testserver to aid in testing of Python's Azure Core package + +It has the following component: + +coretestserver +-------------- + +A testing server for Azure Core tests to use + +Contributing +============ + +This project has adopted the +`Microsoft Open Source Code of Conduct `__. +For more information see the +`Code of Conduct FAQ `__ +or contact +`opencode@microsoft.com `__ +with any additional questions or comments. From c4ac95b7e82a14bb9cfe01dd0aeada08f3a24125 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 25 Jun 2021 12:27:58 -0400 Subject: [PATCH 28/28] ignore readme.rst --- eng/.docsettings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml index f00b7eb0e6ef..2e5778981834 100644 --- a/eng/.docsettings.yml +++ b/eng/.docsettings.yml @@ -94,7 +94,7 @@ known_content_issues: - ['sdk/containerregistry/azure-containerregistry/swagger/README.md', '#4554'] - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554'] - ['sdk/attestation/azure-security-attestation/swagger/README.md', '#4554'] - - ['sdk/core/azure-core/tests/testserver_tests/coretestserver/README.md', '#4554'] + - ['sdk/core/azure-core/tests/testserver_tests/coretestserver/README.rst', '#4554'] # common. - ['sdk/appconfiguration/azure-appconfiguration/README.md', 'common']