From ac864af2bfb103fc37d1b5552cd94be3950dd0d2 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Fri, 18 Jun 2021 19:29:59 +0100 Subject: [PATCH 01/28] Integrate autobahn tests with pytest --- CHANGES/4247.feature | 1 + requirements/dev.txt | 2 ++ tests/autobahn/client/docker-compose.yml | 2 +- tests/autobahn/server/docker-compose.yml | 2 +- tests/autobahn/test_autobahn.py | 32 ++++++++++++++++++++++++ 5 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 CHANGES/4247.feature create mode 100644 tests/autobahn/test_autobahn.py diff --git a/CHANGES/4247.feature b/CHANGES/4247.feature new file mode 100644 index 00000000000..7fe2f339a7b --- /dev/null +++ b/CHANGES/4247.feature @@ -0,0 +1 @@ +Automated running autobahn test suit by integrating with pytest. diff --git a/requirements/dev.txt b/requirements/dev.txt index a2e54a04764..88eee405c46 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -304,3 +304,5 @@ setuptools==51.3.1 # blockdiag # gunicorn # sphinx + +python-on-whales==0.19.0 diff --git a/tests/autobahn/client/docker-compose.yml b/tests/autobahn/client/docker-compose.yml index ac6a8bf3ab7..cc2f3015054 100644 --- a/tests/autobahn/client/docker-compose.yml +++ b/tests/autobahn/client/docker-compose.yml @@ -11,7 +11,7 @@ services: target: /reports aiohttp: - image: aiohttp-autobahn_aiohttp + image: autobahn_aiohttp depends_on: - autobahn command: ["python", "tests/autobahn/client/client.py"] diff --git a/tests/autobahn/server/docker-compose.yml b/tests/autobahn/server/docker-compose.yml index 8f12f2d19cc..36554a6997d 100644 --- a/tests/autobahn/server/docker-compose.yml +++ b/tests/autobahn/server/docker-compose.yml @@ -14,5 +14,5 @@ services: command: ["wstest", "--mode", "fuzzingclient", "--spec", "/config/fuzzingclient.json"] aiohttp: - image: aiohttp-autobahn_aiohttp + image: autobahn_aiohttp command: ["python", "tests/autobahn/server/server.py"] diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py new file mode 100644 index 00000000000..8c2810d4d6d --- /dev/null +++ b/tests/autobahn/test_autobahn.py @@ -0,0 +1,32 @@ +import os +import shutil + +import pytest +from python_on_whales import DockerClient + + +@pytest.fixture(scope="session", autouse=True) +def create_report_directory(): + path = "reports" + if os.path.isdir(path): + shutil.rmtree(path) + os.mkdir(path) + + +@pytest.fixture(scope="session", autouse=True) +def build_aiohttp_docker_image(): + docker = DockerClient(compose_files=["docker-compose.yml"]) + yield docker.compose.build() + docker.compose.rm() + + +def test_client(): + docker = DockerClient(compose_files=["client/docker-compose.yml"]) + docker.compose.up(abort_on_container_exit=True) + docker.compose.down() + + +def test_server(): + docker = DockerClient(compose_files=["server/docker-compose.yml"]) + docker.compose.up(abort_on_container_exit=True) + docker.compose.down() From 3ccc9d8b611e88946cac4a66210587680b8680e0 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Fri, 18 Jun 2021 20:07:56 +0100 Subject: [PATCH 02/28] Fix docker compose file paths --- tests/autobahn/test_autobahn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 8c2810d4d6d..8f462e7638d 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -15,18 +15,18 @@ def create_report_directory(): @pytest.fixture(scope="session", autouse=True) def build_aiohttp_docker_image(): - docker = DockerClient(compose_files=["docker-compose.yml"]) + docker = DockerClient(compose_files=["tests/autobahn/docker-compose.yml"]) yield docker.compose.build() docker.compose.rm() def test_client(): - docker = DockerClient(compose_files=["client/docker-compose.yml"]) + docker = DockerClient(compose_files=["tests/autobahn/client/docker-compose.yml"]) docker.compose.up(abort_on_container_exit=True) docker.compose.down() def test_server(): - docker = DockerClient(compose_files=["server/docker-compose.yml"]) + docker = DockerClient(compose_files=["tests/autobahn/server/docker-compose.yml"]) docker.compose.up(abort_on_container_exit=True) docker.compose.down() From 989ec1381a6346feec9d0a143b98f98abfc45f1e Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 21 Jun 2021 21:11:38 +0100 Subject: [PATCH 03/28] Fix typo in CHANGES file --- CHANGES/4247.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES/4247.feature b/CHANGES/4247.feature index 7fe2f339a7b..86463d0577d 100644 --- a/CHANGES/4247.feature +++ b/CHANGES/4247.feature @@ -1 +1 @@ -Automated running autobahn test suit by integrating with pytest. +Automated running autobahn test suite by integrating with pytest. From aea9cd1cde8dfdea020ac4d1c4341246cc4e0bc0 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 21 Jun 2021 22:31:43 +0100 Subject: [PATCH 04/28] Fix add python-on-whales dependency to .in file instead of .txt --- requirements/dev.in | 1 + requirements/dev.txt | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/dev.in b/requirements/dev.in index 31b14be9997..01a343e7b9a 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -2,3 +2,4 @@ -r test.txt -r doc.txt cherry_picker==2.0.0; python_version>="3.6" +python-on-whales==0.19.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index 88eee405c46..a2e54a04764 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -304,5 +304,3 @@ setuptools==51.3.1 # blockdiag # gunicorn # sphinx - -python-on-whales==0.19.0 From 83e03e07fdf3103841e002a7219dc83d3378c476 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 21 Jun 2021 22:32:37 +0100 Subject: [PATCH 05/28] Use pathlib instead of os --- tests/autobahn/test_autobahn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 8f462e7638d..3f2c40ef7c2 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -1,5 +1,5 @@ -import os import shutil +from pathlib import Path import pytest from python_on_whales import DockerClient @@ -7,10 +7,10 @@ @pytest.fixture(scope="session", autouse=True) def create_report_directory(): - path = "reports" - if os.path.isdir(path): + path = Path("tests/autobahn/reports") + if path.is_dir(): shutil.rmtree(path) - os.mkdir(path) + path.mkdir() @pytest.fixture(scope="session", autouse=True) From aee067dd19d1e314db130fd59d6523dfc13caf6c Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 21 Jun 2021 23:47:03 +0100 Subject: [PATCH 06/28] Use buildx instead of compose build --- tests/autobahn/client/docker-compose.yml | 2 +- tests/autobahn/docker-compose.yml | 6 ------ tests/autobahn/server/docker-compose.yml | 2 +- tests/autobahn/test_autobahn.py | 11 ++++++++--- 4 files changed, 10 insertions(+), 11 deletions(-) delete mode 100644 tests/autobahn/docker-compose.yml diff --git a/tests/autobahn/client/docker-compose.yml b/tests/autobahn/client/docker-compose.yml index cc2f3015054..80b3adf13de 100644 --- a/tests/autobahn/client/docker-compose.yml +++ b/tests/autobahn/client/docker-compose.yml @@ -11,7 +11,7 @@ services: target: /reports aiohttp: - image: autobahn_aiohttp + image: aiohttp depends_on: - autobahn command: ["python", "tests/autobahn/client/client.py"] diff --git a/tests/autobahn/docker-compose.yml b/tests/autobahn/docker-compose.yml deleted file mode 100644 index ea6b640810d..00000000000 --- a/tests/autobahn/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3.9" -services: - aiohttp: - build: - context: ../.. - dockerfile: tests/autobahn/Dockerfile.aiohttp diff --git a/tests/autobahn/server/docker-compose.yml b/tests/autobahn/server/docker-compose.yml index 36554a6997d..aeae7e1fe0c 100644 --- a/tests/autobahn/server/docker-compose.yml +++ b/tests/autobahn/server/docker-compose.yml @@ -14,5 +14,5 @@ services: command: ["wstest", "--mode", "fuzzingclient", "--spec", "/config/fuzzingclient.json"] aiohttp: - image: autobahn_aiohttp + image: aiohttp command: ["python", "tests/autobahn/server/server.py"] diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 3f2c40ef7c2..d4444a3bba8 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -15,9 +15,14 @@ def create_report_directory(): @pytest.fixture(scope="session", autouse=True) def build_aiohttp_docker_image(): - docker = DockerClient(compose_files=["tests/autobahn/docker-compose.yml"]) - yield docker.compose.build() - docker.compose.rm() + docker = DockerClient() + docker.buildx.build( + context_path=".", file="tests/autobahn/Dockerfile.aiohttp", tags=["aiohttp"] + ) + try: + yield + finally: + docker.buildx.remove("default") def test_client(): From b09dab46f39b2a094e0bcb927b326347beb7d39e Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Fri, 25 Jun 2021 18:56:42 +0100 Subject: [PATCH 07/28] Regenerate dev requirements --- requirements/dev.txt | 6 ++++++ tests/autobahn/test_autobahn.py | 1 + 2 files changed, 7 insertions(+) diff --git a/requirements/dev.txt b/requirements/dev.txt index a2e54a04764..0de32b477bb 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -59,6 +59,7 @@ click==7.1.2 # cherry-picker # click-default-group # towncrier + # typer click-default-group==1.2.2 # via towncrier coverage[toml]==6.0 @@ -177,6 +178,8 @@ pycodestyle==2.7.0 # flake8 pycparser==2.20 # via cffi +pydantic==1.8.2 + # via python-on-whales pyflakes==2.3.0 # via # -r requirements/lint.txt @@ -204,6 +207,8 @@ pytest-mock==3.6.1 # via -r requirements/test.txt python-dateutil==2.8.1 # via freezegun +python-on-whales==0.19.0 + # via -r requirements/dev.in pytz==2020.5 # via babel pyyaml==5.4.1 @@ -220,6 +225,7 @@ regex==2020.11.13 requests==2.25.1 # via # cherry-picker + # python-on-whales # sphinx setuptools-git==1.2 # via -r requirements/test.txt diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index d4444a3bba8..9d1933ef759 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -22,6 +22,7 @@ def build_aiohttp_docker_image(): try: yield finally: + # default is the name of the default builder docker.buildx.remove("default") From 0d145997f3f0390dc244b46b4dfa18b8cb9de87e Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Tue, 6 Jul 2021 21:43:14 +0100 Subject: [PATCH 08/28] Rename changes file --- CHANGES/{4247.feature => 4247.1.misc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CHANGES/{4247.feature => 4247.1.misc} (100%) diff --git a/CHANGES/4247.feature b/CHANGES/4247.1.misc similarity index 100% rename from CHANGES/4247.feature rename to CHANGES/4247.1.misc From 4618687137b64a5a0aa59cda216acd86fa65c0f4 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Sat, 10 Jul 2021 00:05:32 +0100 Subject: [PATCH 09/28] Use request fspath instead of hard coded path --- tests/autobahn/test_autobahn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 9d1933ef759..886b762de8e 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -5,9 +5,9 @@ from python_on_whales import DockerClient -@pytest.fixture(scope="session", autouse=True) -def create_report_directory(): - path = Path("tests/autobahn/reports") +@pytest.fixture(scope="function", autouse=True) +def create_report_directory(request): + path = Path(f"{request.fspath.dirname}/reports") if path.is_dir(): shutil.rmtree(path) path.mkdir() From 0a542ac99b8189d5258c93b63b5876ea20115ecd Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Sat, 10 Jul 2021 00:05:56 +0100 Subject: [PATCH 10/28] Create a sepearte builder when building aiohttp --- tests/autobahn/test_autobahn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 886b762de8e..500e9f06226 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -16,14 +16,14 @@ def create_report_directory(request): @pytest.fixture(scope="session", autouse=True) def build_aiohttp_docker_image(): docker = DockerClient() + aiohttp_builder = docker.buildx.create(name="aiohttp", use=True) docker.buildx.build( context_path=".", file="tests/autobahn/Dockerfile.aiohttp", tags=["aiohttp"] ) try: yield finally: - # default is the name of the default builder - docker.buildx.remove("default") + aiohttp_builder.remove() def test_client(): From 52ecc3f923223fc69cd690627a9d5e9f78c9d063 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 19 Jul 2021 12:48:16 +0100 Subject: [PATCH 11/28] Use subprocess instead of python-on-whales --- tests/autobahn/test_autobahn.py | 52 +++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 500e9f06226..a75bc5f0ea3 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -1,8 +1,8 @@ import shutil +import subprocess from pathlib import Path import pytest -from python_on_whales import DockerClient @pytest.fixture(scope="function", autouse=True) @@ -15,24 +15,46 @@ def create_report_directory(request): @pytest.fixture(scope="session", autouse=True) def build_aiohttp_docker_image(): - docker = DockerClient() - aiohttp_builder = docker.buildx.create(name="aiohttp", use=True) - docker.buildx.build( - context_path=".", file="tests/autobahn/Dockerfile.aiohttp", tags=["aiohttp"] + subprocess.run( + [ + "docker", + "build", + "-f", + "tests/autobahn/Dockerfile.aiohttp", + "-t", + "aiohttp", + ".", + ] ) - try: - yield - finally: - aiohttp_builder.remove() def test_client(): - docker = DockerClient(compose_files=["tests/autobahn/client/docker-compose.yml"]) - docker.compose.up(abort_on_container_exit=True) - docker.compose.down() + subprocess.run( + [ + "docker-compose", + "-f", + "tests/autobahn/client/docker-compose.yml", + "up", + "--abort-on-container-exit", + ] + ) + + subprocess.run( + ["docker-compose", "-f", "tests/autobahn/client/docker-compose.yml", "down"] + ) def test_server(): - docker = DockerClient(compose_files=["tests/autobahn/server/docker-compose.yml"]) - docker.compose.up(abort_on_container_exit=True) - docker.compose.down() + subprocess.run( + [ + "docker-compose", + "-f", + "tests/autobahn/server/docker-compose.yml", + "up", + "--abort-on-container-exit", + ] + ) + + subprocess.run( + ["docker-compose", "-f", "tests/autobahn/server/docker-compose.yml", "down"] + ) From b06819a339b1793d2472f081576d16a88845c24f Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 19 Jul 2021 13:02:17 +0100 Subject: [PATCH 12/28] Extract failed tests and make assertions on them --- tests/autobahn/test_autobahn.py | 62 +++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index a75bc5f0ea3..05fd039704e 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -1,3 +1,4 @@ +import json import shutil import subprocess from pathlib import Path @@ -28,7 +29,28 @@ def build_aiohttp_docker_image(): ) -def test_client(): +def get_failed_tests(report_path: str, name) -> list[dict]: + with open(Path(f"{report_path}/index.json")) as f: + result_summary = json.load(f)[name] + failed_messages = [] + PASS = {"OK", "INFORMATIONAL"} + for results in result_summary.values(): + if results["behavior"] not in PASS or results["behaviorClose"] not in PASS: + with open(Path(f"{report_path}/{results['reportfile']}")) as f: + report = json.load(f) + failed_messages.append( + { + "case": report["case"], + "description": report["description"], + "expectation": report["expectation"], + "expected": report["expected"], + "received": report["received"], + } + ) + return failed_messages + + +def test_client() -> None: subprocess.run( [ "docker-compose", @@ -43,8 +65,25 @@ def test_client(): ["docker-compose", "-f", "tests/autobahn/client/docker-compose.yml", "down"] ) + failed_messages = get_failed_tests("tests/autobahn/reports/clients", "aiohttp") + + if failed_messages: + pytest.fail( + "\n".join( + [ + f"case: {msg['case']}" + f"\ndescription: {msg['description']}" + f"\nexpectation: {msg['expectation']}" + f"\nexpected: {msg['expected']}" + f"\nreceived: {msg['received']}" + for msg in failed_messages + ] + ), + pytrace=False, + ) + -def test_server(): +def test_server() -> None: subprocess.run( [ "docker-compose", @@ -58,3 +97,22 @@ def test_server(): subprocess.run( ["docker-compose", "-f", "tests/autobahn/server/docker-compose.yml", "down"] ) + + failed_messages = get_failed_tests( + "tests/autobahn/reports/servers", "AutobahnServer" + ) + + if failed_messages: + pytest.fail( + "\n".join( + [ + f"case: {msg['case']}" + f"\ndescription: {msg['description']}" + f"\nexpectation: {msg['expectation']}" + f"\nexpected: {msg['expected']}" + f"\nreceived: {msg['received']}" + for msg in failed_messages + ] + ), + pytrace=False, + ) From 0ef061e43392815272371817ddacc5b36944a413 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 19 Jul 2021 21:57:44 +0100 Subject: [PATCH 13/28] Fix lint issues --- tests/autobahn/test_autobahn.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 05fd039704e..207dc4a1156 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -2,12 +2,13 @@ import shutil import subprocess from pathlib import Path +from typing import Any, Dict, List import pytest @pytest.fixture(scope="function", autouse=True) -def create_report_directory(request): +def create_report_directory(request: Any) -> None: path = Path(f"{request.fspath.dirname}/reports") if path.is_dir(): shutil.rmtree(path) @@ -15,7 +16,7 @@ def create_report_directory(request): @pytest.fixture(scope="session", autouse=True) -def build_aiohttp_docker_image(): +def build_aiohttp_docker_image() -> None: subprocess.run( [ "docker", @@ -29,7 +30,7 @@ def build_aiohttp_docker_image(): ) -def get_failed_tests(report_path: str, name) -> list[dict]: +def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: with open(Path(f"{report_path}/index.json")) as f: result_summary = json.load(f)[name] failed_messages = [] From 8716e006bdea467be0c3f8b51feaad1535d1154b Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 19 Jul 2021 23:12:22 +0100 Subject: [PATCH 14/28] Fix fixture scope --- tests/autobahn/test_autobahn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 207dc4a1156..637ee6ee1ef 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -7,7 +7,7 @@ import pytest -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="module", autouse=True) def create_report_directory(request: Any) -> None: path = Path(f"{request.fspath.dirname}/reports") if path.is_dir(): @@ -15,7 +15,7 @@ def create_report_directory(request: Any) -> None: path.mkdir() -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="module", autouse=True) def build_aiohttp_docker_image() -> None: subprocess.run( [ From eedd7f100e8304d3836c1131043608c3ecf74cfd Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Tue, 20 Jul 2021 11:11:32 +0100 Subject: [PATCH 15/28] Add ports to docker-compose files --- tests/autobahn/client/docker-compose.yml | 2 ++ tests/autobahn/server/docker-compose.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/autobahn/client/docker-compose.yml b/tests/autobahn/client/docker-compose.yml index 80b3adf13de..9937b76b9c6 100644 --- a/tests/autobahn/client/docker-compose.yml +++ b/tests/autobahn/client/docker-compose.yml @@ -9,6 +9,8 @@ services: - type: bind source: ../reports target: /reports + ports: + - "9001:9001" aiohttp: image: aiohttp diff --git a/tests/autobahn/server/docker-compose.yml b/tests/autobahn/server/docker-compose.yml index aeae7e1fe0c..c57d560c50c 100644 --- a/tests/autobahn/server/docker-compose.yml +++ b/tests/autobahn/server/docker-compose.yml @@ -16,3 +16,5 @@ services: aiohttp: image: aiohttp command: ["python", "tests/autobahn/server/server.py"] + ports: + - "9001:9001" From 43e04e02b76fe9eb67517426422f887871da8994 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Tue, 20 Jul 2021 11:40:09 +0100 Subject: [PATCH 16/28] Add wait-for-it package --- tests/autobahn/client/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/autobahn/client/docker-compose.yml b/tests/autobahn/client/docker-compose.yml index 9937b76b9c6..1d2bad1732c 100644 --- a/tests/autobahn/client/docker-compose.yml +++ b/tests/autobahn/client/docker-compose.yml @@ -16,4 +16,4 @@ services: image: aiohttp depends_on: - autobahn - command: ["python", "tests/autobahn/client/client.py"] + command: ["wait-for-it", "-s", "autobahn:9001", "--", "python", "tests/autobahn/client/client.py"] From 14a678b809a36e838562438c91525c0904e9fa5b Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Tue, 20 Jul 2021 13:22:08 +0100 Subject: [PATCH 17/28] Use xfail instead of fail --- tests/autobahn/test_autobahn.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 637ee6ee1ef..29e1a1b4566 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -69,7 +69,7 @@ def test_client() -> None: failed_messages = get_failed_tests("tests/autobahn/reports/clients", "aiohttp") if failed_messages: - pytest.fail( + pytest.xfail( "\n".join( [ f"case: {msg['case']}" @@ -79,8 +79,7 @@ def test_client() -> None: f"\nreceived: {msg['received']}" for msg in failed_messages ] - ), - pytrace=False, + ) ) @@ -104,7 +103,7 @@ def test_server() -> None: ) if failed_messages: - pytest.fail( + pytest.xfail( "\n".join( [ f"case: {msg['case']}" @@ -114,6 +113,5 @@ def test_server() -> None: f"\nreceived: {msg['received']}" for msg in failed_messages ] - ), - pytrace=False, + ) ) From cf2a53b070dbef19eae463ba8da9fe71cdd4944f Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Tue, 20 Jul 2021 23:25:34 +0100 Subject: [PATCH 18/28] Use wstest cmd tool instead of the docker image --- tests/autobahn/client/client.py | 2 +- tests/autobahn/client/fuzzingserver.json | 2 +- tests/autobahn/server/fuzzingclient.json | 4 +- tests/autobahn/server/server.py | 5 +- tests/autobahn/test_autobahn.py | 121 ++++++++++++++++------- 5 files changed, 90 insertions(+), 44 deletions(-) diff --git a/tests/autobahn/client/client.py b/tests/autobahn/client/client.py index 107c183070e..dfca77d12b2 100644 --- a/tests/autobahn/client/client.py +++ b/tests/autobahn/client/client.py @@ -38,4 +38,4 @@ async def run(url: str, name: str) -> None: if __name__ == "__main__": - asyncio.run(run("http://autobahn:9001", "aiohttp")) + asyncio.run(run("http://localhost:9001", "aiohttp")) diff --git a/tests/autobahn/client/fuzzingserver.json b/tests/autobahn/client/fuzzingserver.json index 5f8bf31d203..04986b6126c 100644 --- a/tests/autobahn/client/fuzzingserver.json +++ b/tests/autobahn/client/fuzzingserver.json @@ -3,7 +3,7 @@ "url": "ws://localhost:9001", "options": {"failByDrop": false}, - "outdir": "./reports/clients", + "outdir": "./tests/autobahn/reports/clients", "webport": 8080, "cases": ["*"], diff --git a/tests/autobahn/server/fuzzingclient.json b/tests/autobahn/server/fuzzingclient.json index e9bef9591dc..a4794baefc9 100644 --- a/tests/autobahn/server/fuzzingclient.json +++ b/tests/autobahn/server/fuzzingclient.json @@ -1,11 +1,11 @@ { "options": { "failByDrop": false }, - "outdir": "./reports/servers", + "outdir": "./tests/autobahn/reports/servers", "servers": [ { "agent": "AutobahnServer", - "url": "ws://aiohttp:9001", + "url": "ws://localhost:9001", "options": { "version": 18 } } ], diff --git a/tests/autobahn/server/server.py b/tests/autobahn/server/server.py index d4ca04b1d5f..32a16948158 100644 --- a/tests/autobahn/server/server.py +++ b/tests/autobahn/server/server.py @@ -13,9 +13,9 @@ async def wshandler(request: web.Request) -> web.WebSocketResponse: await ws.prepare(request) - while True: - msg = await ws.receive() + request.app["websockets"].append(ws) + for msg in ws: if msg.type == web.WSMsgType.TEXT: await ws.send_str(msg.data) elif msg.type == web.WSMsgType.BINARY: @@ -40,6 +40,7 @@ async def on_shutdown(app: web.Application) -> None: ) app = web.Application() + app["websockets"] = [] app.router.add_route("GET", "/", wshandler) app.on_shutdown.append(on_shutdown) try: diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 29e1a1b4566..98a5c48bb62 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -1,11 +1,17 @@ import json import shutil +import socket import subprocess +import sys +import time from pathlib import Path -from typing import Any, Dict, List +from typing import Any import pytest +HOST = "localhost" +PORT = 9001 + @pytest.fixture(scope="module", autouse=True) def create_report_directory(request: Any) -> None: @@ -15,22 +21,39 @@ def create_report_directory(request: Any) -> None: path.mkdir() -@pytest.fixture(scope="module", autouse=True) -def build_aiohttp_docker_image() -> None: - subprocess.run( - [ - "docker", - "build", - "-f", - "tests/autobahn/Dockerfile.aiohttp", - "-t", - "aiohttp", - ".", - ] - ) +@pytest.fixture(scope="session", autouse=True) +def setup_venv() -> None: + if not Path("tests/autobahn/autobahntestsuite-env").exists(): + print("Creating Python 2.7 environment and installing autobahntestsuite") + subprocess.check_call( + ["virtualenv", "-p", "python2.7", "tests/autobahn/autobahntestsuite-env"] + ) + subprocess.check_call( + [ + "tests/autobahn/autobahntestsuite-env/bin/pip", + "install", + "autobahntestsuite>=0.8.0", + ] + ) + + +def wait_for_server(host: str, port: int) -> None: + while True: + sock = socket.socket() + try: + sock.connect((HOST, PORT)) + except OSError as sock_err: + if sock_err.errno == socket.errno.ECONNREFUSED: + time.sleep(0.01) + else: + raise + else: + return + finally: + sock.close() -def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: +def get_failed_tests(report_path: str, name: str): with open(Path(f"{report_path}/index.json")) as f: result_summary = json.load(f)[name] failed_messages = [] @@ -52,19 +75,30 @@ def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: def test_client() -> None: - subprocess.run( + print("Starting autobahntestsuite server") + server = subprocess.Popen( [ - "docker-compose", - "-f", - "tests/autobahn/client/docker-compose.yml", - "up", - "--abort-on-container-exit", + "tests/autobahn/autobahntestsuite-env/bin/wstest", + "-m", + "fuzzingserver", + "-s", + "tests/autobahn/client/fuzzingserver.json", ] ) - - subprocess.run( - ["docker-compose", "-f", "tests/autobahn/client/docker-compose.yml", "down"] - ) + print("Waiting for server to start") + wait_for_server(HOST, PORT) + try: + print("Running wsproto test client") + client = subprocess.Popen( + [sys.executable] + ["tests/autobahn/client/client.py"] + ) + client.wait() + finally: + print("Stopping server") + client.terminate() + client.wait() + server.terminate() + server.wait() failed_messages = get_failed_tests("tests/autobahn/reports/clients", "aiohttp") @@ -84,19 +118,30 @@ def test_client() -> None: def test_server() -> None: - subprocess.run( - [ - "docker-compose", - "-f", - "tests/autobahn/server/docker-compose.yml", - "up", - "--abort-on-container-exit", - ] - ) - - subprocess.run( - ["docker-compose", "-f", "tests/autobahn/server/docker-compose.yml", "down"] - ) + print("Starting wsproto test server") + server = subprocess.Popen([sys.executable] + ["tests/autobahn/server/server.py"]) + try: + print("Waiting for server to start") + wait_for_server(HOST, PORT) + print("Starting autobahntestsuite client") + client = subprocess.Popen( + [ + "tests/autobahn/autobahntestsuite-env/bin/wstest", + "-m", + "fuzzingclient", + "-s", + "tests/autobahn/server/fuzzingclient.json", + "-o", + "tests/autobahn/reports/servers", + ] + ) + client.wait() + finally: + print("Stopping server") + client.terminate() + client.wait() + server.terminate() + server.wait() failed_messages = get_failed_tests( "tests/autobahn/reports/servers", "AutobahnServer" From 48335a598eb433560f6aa0dc68a6c9c92a1dacf5 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Tue, 20 Jul 2021 23:37:31 +0100 Subject: [PATCH 19/28] Fix lint issues --- tests/autobahn/server/server.py | 4 +++- tests/autobahn/test_autobahn.py | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/autobahn/server/server.py b/tests/autobahn/server/server.py index 32a16948158..c0e50259b47 100644 --- a/tests/autobahn/server/server.py +++ b/tests/autobahn/server/server.py @@ -15,7 +15,9 @@ async def wshandler(request: web.Request) -> web.WebSocketResponse: request.app["websockets"].append(ws) - for msg in ws: + while True: + msg = await ws.receive() + if msg.type == web.WSMsgType.TEXT: await ws.send_str(msg.data) elif msg.type == web.WSMsgType.BINARY: diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 98a5c48bb62..1dd0d026054 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -1,3 +1,4 @@ +import errno import json import shutil import socket @@ -5,7 +6,7 @@ import sys import time from pathlib import Path -from typing import Any +from typing import Any, Dict, List import pytest @@ -43,7 +44,7 @@ def wait_for_server(host: str, port: int) -> None: try: sock.connect((HOST, PORT)) except OSError as sock_err: - if sock_err.errno == socket.errno.ECONNREFUSED: + if sock_err.errno == errno.ECONNREFUSED: time.sleep(0.01) else: raise @@ -53,7 +54,7 @@ def wait_for_server(host: str, port: int) -> None: sock.close() -def get_failed_tests(report_path: str, name: str): +def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: with open(Path(f"{report_path}/index.json")) as f: result_summary = json.load(f)[name] failed_messages = [] From bb91f0e937b8e7e25344748f2316a607a885e17a Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Wed, 21 Jul 2021 12:57:16 +0100 Subject: [PATCH 20/28] Use assert statement with custom output Co-authored-by: Sviatoslav Sydorenko --- tests/autobahn/test_autobahn.py | 36 +++++++++++---------------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 1dd0d026054..71e1d247cda 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -103,19 +103,13 @@ def test_client() -> None: failed_messages = get_failed_tests("tests/autobahn/reports/clients", "aiohttp") - if failed_messages: - pytest.xfail( - "\n".join( - [ - f"case: {msg['case']}" - f"\ndescription: {msg['description']}" - f"\nexpectation: {msg['expectation']}" - f"\nexpected: {msg['expected']}" - f"\nreceived: {msg['received']}" - for msg in failed_messages - ] - ) + assert not failed_messages, "\n".join( + "\n\t".join( + f"{field}: {msg[field]}" + for field in ("case", "description", "expectation", "expected", "received") ) + for msg in failed_messages + ) def test_server() -> None: @@ -148,16 +142,10 @@ def test_server() -> None: "tests/autobahn/reports/servers", "AutobahnServer" ) - if failed_messages: - pytest.xfail( - "\n".join( - [ - f"case: {msg['case']}" - f"\ndescription: {msg['description']}" - f"\nexpectation: {msg['expectation']}" - f"\nexpected: {msg['expected']}" - f"\nreceived: {msg['received']}" - for msg in failed_messages - ] - ) + assert not failed_messages, "\n".join( + "\n\t".join( + f"{field}: {msg[field]}" + for field in ("case", "description", "expectation", "expected", "received") ) + for msg in failed_messages + ) From bc4b61f4669b78a2523b6a918dfb62be52653dfd Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Wed, 21 Jul 2021 13:03:39 +0100 Subject: [PATCH 21/28] Code cleanup --- tests/autobahn/test_autobahn.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 71e1d247cda..8928739d753 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -1,4 +1,3 @@ -import errno import json import shutil import socket @@ -43,11 +42,8 @@ def wait_for_server(host: str, port: int) -> None: sock = socket.socket() try: sock.connect((HOST, PORT)) - except OSError as sock_err: - if sock_err.errno == errno.ECONNREFUSED: - time.sleep(0.01) - else: - raise + except ConnectionRefusedError: + time.sleep(0.01) else: return finally: @@ -55,23 +51,16 @@ def wait_for_server(host: str, port: int) -> None: def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: - with open(Path(f"{report_path}/index.json")) as f: - result_summary = json.load(f)[name] + report_path = Path(report_path) + result_summary = json.loads((report_path / "index.json").read_text())[name] failed_messages = [] PASS = {"OK", "INFORMATIONAL"} + entry_fields = {"case", "description", "expectation", "expected", "received"} for results in result_summary.values(): - if results["behavior"] not in PASS or results["behaviorClose"] not in PASS: - with open(Path(f"{report_path}/{results['reportfile']}")) as f: - report = json.load(f) - failed_messages.append( - { - "case": report["case"], - "description": report["description"], - "expectation": report["expectation"], - "expected": report["expected"], - "received": report["received"], - } - ) + if results["behavior"] in PASS and results["behaviorClose"] in PASS: + continue + report = json.loads((report_path / results["reportfile"]).read_text()) + failed_messages.append({field: report[field] for field in entry_fields}) return failed_messages From 23536f4054b997077e78b2ddb5495f11468c0649 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Wed, 21 Jul 2021 20:43:23 +0100 Subject: [PATCH 22/28] Use docker instead of docker-compose --- .mypy.ini | 3 + requirements/dev.in | 1 + requirements/dev.txt | 3 + tests/autobahn/Dockerfile.autobahn | 6 + tests/autobahn/client/docker-compose.yml | 19 ---- tests/autobahn/client/fuzzingserver.json | 2 +- tests/autobahn/run-tests.sh | 12 -- tests/autobahn/server/docker-compose.yml | 20 ---- tests/autobahn/server/fuzzingclient.json | 2 +- tests/autobahn/test_autobahn.py | 136 +++++++++++------------ 10 files changed, 78 insertions(+), 126 deletions(-) create mode 100644 tests/autobahn/Dockerfile.autobahn delete mode 100644 tests/autobahn/client/docker-compose.yml delete mode 100755 tests/autobahn/run-tests.sh delete mode 100644 tests/autobahn/server/docker-compose.yml diff --git a/.mypy.ini b/.mypy.ini index 5b5796e4a51..ebcd461441c 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -41,3 +41,6 @@ ignore_missing_imports = True [mypy-uvloop] ignore_missing_imports = True + +[mypy-python_on_whales] +ignore_missing_imports = True diff --git a/requirements/dev.in b/requirements/dev.in index 01a343e7b9a..d827d37d65f 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -3,3 +3,4 @@ -r doc.txt cherry_picker==2.0.0; python_version>="3.6" python-on-whales==0.19.0 +wait-for-it==2.2.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index 0de32b477bb..6498856e84c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -60,6 +60,7 @@ click==7.1.2 # click-default-group # towncrier # typer + # wait-for-it click-default-group==1.2.2 # via towncrier coverage[toml]==6.0 @@ -299,6 +300,8 @@ virtualenv==20.4.2 # via # -r requirements/lint.txt # pre-commit +wait-for-it==2.2.0 + # via -r requirements/dev.in webcolors==1.11.1 # via blockdiag yarl==1.6.3 diff --git a/tests/autobahn/Dockerfile.autobahn b/tests/autobahn/Dockerfile.autobahn new file mode 100644 index 00000000000..45f18182804 --- /dev/null +++ b/tests/autobahn/Dockerfile.autobahn @@ -0,0 +1,6 @@ +FROM crossbario/autobahn-testsuite:0.8.2 + +RUN apt-get update && apt-get install python3 python3-pip -y +RUN pip3 install wait-for-it + +CMD ["wstest", "--mode", "fuzzingserver", "--spec", "/config/fuzzingserver.json"] diff --git a/tests/autobahn/client/docker-compose.yml b/tests/autobahn/client/docker-compose.yml deleted file mode 100644 index 1d2bad1732c..00000000000 --- a/tests/autobahn/client/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: "3.9" -services: - autobahn: - image: crossbario/autobahn-testsuite:0.8.2 - volumes: - - type: bind - source: ./fuzzingserver.json - target: /config/fuzzingserver.json - - type: bind - source: ../reports - target: /reports - ports: - - "9001:9001" - - aiohttp: - image: aiohttp - depends_on: - - autobahn - command: ["wait-for-it", "-s", "autobahn:9001", "--", "python", "tests/autobahn/client/client.py"] diff --git a/tests/autobahn/client/fuzzingserver.json b/tests/autobahn/client/fuzzingserver.json index 04986b6126c..5f8bf31d203 100644 --- a/tests/autobahn/client/fuzzingserver.json +++ b/tests/autobahn/client/fuzzingserver.json @@ -3,7 +3,7 @@ "url": "ws://localhost:9001", "options": {"failByDrop": false}, - "outdir": "./tests/autobahn/reports/clients", + "outdir": "./reports/clients", "webport": 8080, "cases": ["*"], diff --git a/tests/autobahn/run-tests.sh b/tests/autobahn/run-tests.sh deleted file mode 100755 index d48894d8cb8..00000000000 --- a/tests/autobahn/run-tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -rm -rf $PWD/reports -mkdir $PWD/reports - -docker-compose -p aiohttp-autobahn build - -docker-compose -f $PWD/client/docker-compose.yml up --abort-on-container-exit -docker-compose -f $PWD/client/docker-compose.yml down - -docker-compose -f $PWD/server/docker-compose.yml up --abort-on-container-exit -docker-compose -f $PWD/server/docker-compose.yml down diff --git a/tests/autobahn/server/docker-compose.yml b/tests/autobahn/server/docker-compose.yml deleted file mode 100644 index c57d560c50c..00000000000 --- a/tests/autobahn/server/docker-compose.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: "3.9" -services: - autobahn: - image: crossbario/autobahn-testsuite:0.8.2 - depends_on: - - aiohttp - volumes: - - type: bind - source: ./fuzzingclient.json - target: /config/fuzzingclient.json - - type: bind - source: ../reports - target: /reports - command: ["wstest", "--mode", "fuzzingclient", "--spec", "/config/fuzzingclient.json"] - - aiohttp: - image: aiohttp - command: ["python", "tests/autobahn/server/server.py"] - ports: - - "9001:9001" diff --git a/tests/autobahn/server/fuzzingclient.json b/tests/autobahn/server/fuzzingclient.json index a4794baefc9..0ed2f84acf8 100644 --- a/tests/autobahn/server/fuzzingclient.json +++ b/tests/autobahn/server/fuzzingclient.json @@ -1,6 +1,6 @@ { "options": { "failByDrop": false }, - "outdir": "./tests/autobahn/reports/servers", + "outdir": "./reports/servers", "servers": [ { diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 8928739d753..dcd21754b15 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -1,94 +1,76 @@ import json import shutil -import socket import subprocess import sys -import time from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, Generator, List import pytest - -HOST = "localhost" -PORT = 9001 +from python_on_whales import docker @pytest.fixture(scope="module", autouse=True) def create_report_directory(request: Any) -> None: - path = Path(f"{request.fspath.dirname}/reports") + path = Path(request.fspath.dirname) / "reports" if path.is_dir(): shutil.rmtree(path) path.mkdir() @pytest.fixture(scope="session", autouse=True) -def setup_venv() -> None: - if not Path("tests/autobahn/autobahntestsuite-env").exists(): - print("Creating Python 2.7 environment and installing autobahntestsuite") - subprocess.check_call( - ["virtualenv", "-p", "python2.7", "tests/autobahn/autobahntestsuite-env"] - ) - subprocess.check_call( - [ - "tests/autobahn/autobahntestsuite-env/bin/pip", - "install", - "autobahntestsuite>=0.8.0", - ] - ) - - -def wait_for_server(host: str, port: int) -> None: - while True: - sock = socket.socket() - try: - sock.connect((HOST, PORT)) - except ConnectionRefusedError: - time.sleep(0.01) - else: - return - finally: - sock.close() +def build_autobahn_testsuite() -> Generator[None, None, None]: + docker.build( + file="tests/autobahn/Dockerfile.autobahn", + tags=["autobahn-testsuite"], + context_path=".", + ) + try: + yield + finally: + docker.image.remove(x="autobahn-testsuite") def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: - report_path = Path(report_path) - result_summary = json.loads((report_path / "index.json").read_text())[name] + path = Path(report_path) + result_summary = json.loads((path / "index.json").read_text())[name] failed_messages = [] PASS = {"OK", "INFORMATIONAL"} entry_fields = {"case", "description", "expectation", "expected", "received"} for results in result_summary.values(): if results["behavior"] in PASS and results["behaviorClose"] in PASS: continue - report = json.loads((report_path / results["reportfile"]).read_text()) + report = json.loads((path / results["reportfile"]).read_text()) failed_messages.append({field: report[field] for field in entry_fields}) return failed_messages -def test_client() -> None: - print("Starting autobahntestsuite server") - server = subprocess.Popen( - [ - "tests/autobahn/autobahntestsuite-env/bin/wstest", - "-m", - "fuzzingserver", - "-s", - "tests/autobahn/client/fuzzingserver.json", - ] - ) - print("Waiting for server to start") - wait_for_server(HOST, PORT) +@pytest.mark.skipif(sys.platform != "linux", reason="run only on Linux") +def test_client(request: Any) -> None: try: - print("Running wsproto test client") + print("Starting autobahn-testsuite server") + autobahn_container = docker.run( + detach=True, + image="autobahn-testsuite", + name="autobahn", + publish=[(9001, 9001)], + remove=True, + volumes=[ + (f"{request.fspath.dirname}/client", "/config"), + (f"{request.fspath.dirname}/reports", "/reports"), + ], + ) + print("Running aiohttp test client") client = subprocess.Popen( - [sys.executable] + ["tests/autobahn/client/client.py"] + ["wait-for-it", "-s", "localhost:9001", "--"] + + [sys.executable] + + ["tests/autobahn/client/client.py"] ) client.wait() finally: - print("Stopping server") + print("Stopping client and server") client.terminate() client.wait() - server.terminate() - server.wait() + autobahn_container.stop() failed_messages = get_failed_tests("tests/autobahn/reports/clients", "aiohttp") @@ -101,29 +83,37 @@ def test_client() -> None: ) -def test_server() -> None: - print("Starting wsproto test server") - server = subprocess.Popen([sys.executable] + ["tests/autobahn/server/server.py"]) +@pytest.mark.skipif(sys.platform != "linux", reason="run only on Linux") +def test_server(request: Any) -> None: try: - print("Waiting for server to start") - wait_for_server(HOST, PORT) - print("Starting autobahntestsuite client") - client = subprocess.Popen( - [ - "tests/autobahn/autobahntestsuite-env/bin/wstest", - "-m", - "fuzzingclient", + print("Starting aiohttp test server") + server = subprocess.Popen( + [sys.executable] + ["tests/autobahn/server/server.py"] + ) + print("Starting autobahn-testsuite client") + docker.run( + image="autobahn-testsuite", + name="autobahn", + remove=True, + volumes=[ + (f"{request.fspath.dirname}/server", "/config"), + (f"{request.fspath.dirname}/reports", "/reports"), + ], + networks=["host"], + command=[ + "wait-for-it", "-s", - "tests/autobahn/server/fuzzingclient.json", - "-o", - "tests/autobahn/reports/servers", - ] + "localhost:9001", + "--", + "wstest", + "--mode", + "fuzzingclient", + "--spec", + "/config/fuzzingclient.json", + ], ) - client.wait() finally: - print("Stopping server") - client.terminate() - client.wait() + print("Stopping client and server") server.terminate() server.wait() From 762163473f4524f95f783b55e129d86cbc8807b9 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Wed, 21 Jul 2021 21:40:14 +0100 Subject: [PATCH 23/28] Add xfail decorator --- tests/autobahn/test_autobahn.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index dcd21754b15..28a1d40ac6e 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -45,6 +45,7 @@ def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: @pytest.mark.skipif(sys.platform != "linux", reason="run only on Linux") +@pytest.mark.xfail def test_client(request: Any) -> None: try: print("Starting autobahn-testsuite server") @@ -84,6 +85,7 @@ def test_client(request: Any) -> None: @pytest.mark.skipif(sys.platform != "linux", reason="run only on Linux") +@pytest.mark.xfail def test_server(request: Any) -> None: try: print("Starting aiohttp test server") From 276690239123ea87ded7c518deff1687b639e275 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Thu, 22 Jul 2021 13:15:28 +0100 Subject: [PATCH 24/28] Add tmp_path --- tests/autobahn/test_autobahn.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 28a1d40ac6e..f75b7796b9c 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -1,20 +1,17 @@ import json -import shutil import subprocess import sys from pathlib import Path from typing import Any, Dict, Generator, List import pytest +from pytest import TempPathFactory from python_on_whales import docker -@pytest.fixture(scope="module", autouse=True) -def create_report_directory(request: Any) -> None: - path = Path(request.fspath.dirname) / "reports" - if path.is_dir(): - shutil.rmtree(path) - path.mkdir() +@pytest.fixture(scope="session") +def report_dir(tmp_path_factory: TempPathFactory) -> Path: + return tmp_path_factory.mktemp("reports") @pytest.fixture(scope="session", autouse=True) @@ -46,7 +43,7 @@ def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: @pytest.mark.skipif(sys.platform != "linux", reason="run only on Linux") @pytest.mark.xfail -def test_client(request: Any) -> None: +def test_client(report_dir: Path, request: Any) -> None: try: print("Starting autobahn-testsuite server") autobahn_container = docker.run( @@ -57,7 +54,7 @@ def test_client(request: Any) -> None: remove=True, volumes=[ (f"{request.fspath.dirname}/client", "/config"), - (f"{request.fspath.dirname}/reports", "/reports"), + (f"{report_dir}", "/reports"), ], ) print("Running aiohttp test client") @@ -73,7 +70,7 @@ def test_client(request: Any) -> None: client.wait() autobahn_container.stop() - failed_messages = get_failed_tests("tests/autobahn/reports/clients", "aiohttp") + failed_messages = get_failed_tests(f"{report_dir}/clients", "aiohttp") assert not failed_messages, "\n".join( "\n\t".join( @@ -86,7 +83,7 @@ def test_client(request: Any) -> None: @pytest.mark.skipif(sys.platform != "linux", reason="run only on Linux") @pytest.mark.xfail -def test_server(request: Any) -> None: +def test_server(report_dir: Path, request: Any) -> None: try: print("Starting aiohttp test server") server = subprocess.Popen( @@ -99,7 +96,7 @@ def test_server(request: Any) -> None: remove=True, volumes=[ (f"{request.fspath.dirname}/server", "/config"), - (f"{request.fspath.dirname}/reports", "/reports"), + (f"{report_dir}", "/reports"), ], networks=["host"], command=[ @@ -119,9 +116,7 @@ def test_server(request: Any) -> None: server.terminate() server.wait() - failed_messages = get_failed_tests( - "tests/autobahn/reports/servers", "AutobahnServer" - ) + failed_messages = get_failed_tests(f"{report_dir}/servers", "AutobahnServer") assert not failed_messages, "\n".join( "\n\t".join( From 9d8832512d7a6436123b06ae13a67a5946faa499 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 26 Jul 2021 20:59:50 +0100 Subject: [PATCH 25/28] Remove gitignore --- tests/autobahn/.gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/autobahn/.gitignore diff --git a/tests/autobahn/.gitignore b/tests/autobahn/.gitignore deleted file mode 100644 index 08ab34c5253..00000000000 --- a/tests/autobahn/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/reports From 18dde32548a268b309249dc7fc38d346d114c781 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Mon, 26 Jul 2021 21:00:08 +0100 Subject: [PATCH 26/28] Skip tests only on macOS --- tests/autobahn/test_autobahn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index f75b7796b9c..3b4071da415 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -41,7 +41,7 @@ def get_failed_tests(report_path: str, name: str) -> List[Dict[str, Any]]: return failed_messages -@pytest.mark.skipif(sys.platform != "linux", reason="run only on Linux") +@pytest.mark.skipif(sys.platform == "darwin", reason="Don't run on macOS") @pytest.mark.xfail def test_client(report_dir: Path, request: Any) -> None: try: @@ -81,7 +81,7 @@ def test_client(report_dir: Path, request: Any) -> None: ) -@pytest.mark.skipif(sys.platform != "linux", reason="run only on Linux") +@pytest.mark.skipif(sys.platform == "darwin", reason="Don't run on macOS") @pytest.mark.xfail def test_server(report_dir: Path, request: Any) -> None: try: From 55c59b2d424ccc0017c46e7410475755c7103fd8 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Fri, 13 Aug 2021 19:24:48 +0100 Subject: [PATCH 27/28] Check if docker is available --- tests/autobahn/test_autobahn.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/autobahn/test_autobahn.py b/tests/autobahn/test_autobahn.py index 3b4071da415..5d72e37a17a 100644 --- a/tests/autobahn/test_autobahn.py +++ b/tests/autobahn/test_autobahn.py @@ -6,7 +6,7 @@ import pytest from pytest import TempPathFactory -from python_on_whales import docker +from python_on_whales import DockerException, docker @pytest.fixture(scope="session") @@ -16,11 +16,16 @@ def report_dir(tmp_path_factory: TempPathFactory) -> Path: @pytest.fixture(scope="session", autouse=True) def build_autobahn_testsuite() -> Generator[None, None, None]: - docker.build( - file="tests/autobahn/Dockerfile.autobahn", - tags=["autobahn-testsuite"], - context_path=".", - ) + + try: + docker.build( + file="tests/autobahn/Dockerfile.autobahn", + tags=["autobahn-testsuite"], + context_path=".", + ) + except DockerException: + pytest.skip(msg="The docker daemon is not running.") + try: yield finally: From c8efa785d3d16de7c902fe3623515ebd4dda2d40 Mon Sep 17 00:00:00 2001 From: Anes Abismail Date: Thu, 16 Sep 2021 19:58:20 +0100 Subject: [PATCH 28/28] Regenerate dev.txt --- requirements/dev.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requirements/dev.txt b/requirements/dev.txt index 6498856e84c..57f7f360845 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -279,8 +279,12 @@ towncrier==21.3.0 # via # -r requirements/doc.txt # sphinxcontrib-towncrier +tqdm==4.62.2 + # via python-on-whales trustme==0.9.0 ; platform_machine != "i686" # via -r requirements/test.txt +typer==0.4.0 + # via python-on-whales types-chardet==0.1.3 # via # -r requirements/lint.txt @@ -292,6 +296,7 @@ typing-extensions==3.7.4.3 # async-timeout # mypy # proxy.py + # pydantic uritemplate==3.0.1 # via gidgethub urllib3==1.26.5