From 761e155af50920257c5e61066c39d587675c5148 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 3 Apr 2021 00:19:40 -0400 Subject: [PATCH 1/6] [wasm][tests] Add properties to allow passing args to xharness New properties: - `$(WasmXHarnessArgs)` - `$(WasmXHarnessMonoArgs)` - `$(WasmTestAppArgs)` Xharness command line is built with these: `dotnet xharness wasm test .. $(WasmXHarnessArgs) .. -- $(WasmXHarnessMonoArgs) .. --run .. $(WasmTestAppArgs)` --- .../testing/libraries/testing-wasm.md | 12 ++++++++++ eng/testing/tests.mobile.targets | 24 +++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/workflow/testing/libraries/testing-wasm.md b/docs/workflow/testing/libraries/testing-wasm.md index 3979f9033a09b..242e6c9cee4a9 100644 --- a/docs/workflow/testing/libraries/testing-wasm.md +++ b/docs/workflow/testing/libraries/testing-wasm.md @@ -96,6 +96,18 @@ The following shows how to run tests for a specific library make -C src/mono/wasm/ run-browser-tests-System.AppContext ``` +### Passing arguments to xharness + +- `$(WasmXHarnessArgs)` - xharness command arguments + + Example: `WasmXHarnessArgs="--xyz"` -> becomes `dotnet xharness wasm test --xyz` + +- `$(WasmXHarnessMonoArgs)` - arguments to mono + + Example: `WasmXHarnessMonoArgs="--runtime-arg=--trace=E"` + +- `$(WasmTestAppArgs)` - arguments for the test app itself + ### Running outer loop tests using Browser instance To run all tests, including "outer loop" tests (which are typically slower and in some test suites less reliable, but which are more comprehensive): diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index 81246339652ca..dbb2ced7861b7 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -4,8 +4,6 @@ $([MSBuild]::NormalizeDirectory('$(OutDir)', 'AppBundle')) $([MSBuild]::NormalizePath('$(BundleDir)', '$(RunScriptOutputName)')) true - V8 - $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 true true @@ -13,10 +11,26 @@ true + + + V8 + $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 + + <_XHarnessArgs>wasm $XHARNESS_COMMAND --app=. --output-directory=$XHARNESS_OUT + + <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js + <_XHarnessArgs Condition="'$(IsFunctionalTest)' == 'true'" >$(_XHarnessArgs) --expected-exit-code=$(ExpectedExitCode) + <_XHarnessArgs Condition="'$(WasmXHarnessArgs)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgs) %24WasmXHarnessArgs + + <_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(Scenario)' != 'BuildWasmApps'">--run WasmTestRunner.dll $(AssemblyName).dll + <_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll --testing + + <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) %24WasmTestAppArgs + + $HARNESS_RUNNER $(_XHarnessArgs) -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) + + - - $HARNESS_RUNNER wasm $XHARNESS_COMMAND --app=. --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js --output-directory=$XHARNESS_OUT -- $(RunTestsJSArguments) --run WasmTestRunner.dll $(AssemblyName).dll - $HARNESS_RUNNER wasm $XHARNESS_COMMAND --app=. --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js --output-directory=$XHARNESS_OUT --expected-exit-code=$(ExpectedExitCode) -- $(RunTestsJSArguments) --run $(AssemblyName).dll --testing false true false From a7b4f0bc1af2b4b8f4a3898eabaa2743b75f7a22 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 3 Apr 2021 01:07:35 -0400 Subject: [PATCH 2/6] [wasm][tests] extract wasm bits into tests.wasm.targets --- eng/testing/tests.mobile.targets | 79 +------------------------------- eng/testing/tests.wasm.targets | 78 +++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 77 deletions(-) create mode 100644 eng/testing/tests.wasm.targets diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index dbb2ced7861b7..a0e795f8f7c82 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -11,33 +11,6 @@ true - - - V8 - $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 - - <_XHarnessArgs>wasm $XHARNESS_COMMAND --app=. --output-directory=$XHARNESS_OUT - - <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js - <_XHarnessArgs Condition="'$(IsFunctionalTest)' == 'true'" >$(_XHarnessArgs) --expected-exit-code=$(ExpectedExitCode) - <_XHarnessArgs Condition="'$(WasmXHarnessArgs)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgs) %24WasmXHarnessArgs - - <_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(Scenario)' != 'BuildWasmApps'">--run WasmTestRunner.dll $(AssemblyName).dll - <_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll --testing - - <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) %24WasmTestAppArgs - - $HARNESS_RUNNER $(_XHarnessArgs) -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) - - - - false - true - false - false - false - - false false @@ -81,6 +54,8 @@ <_MobileIntermediateOutputPath>$(IntermediateOutputPath)mobile + + - - - - - PrepareForWasmBuildApp;$(WasmBuildAppDependsOn) - $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'mono', 'wasm', 'emsdk')) - - - - - - - - - - - - - - - - - - - $(BundleDir) - WasmTestRunner.dll - $(MonoProjectRoot)\wasm\runtime-test.js - $(InvariantGlobalization) - true - false - - - - - - $([System.IO.Directory]::GetParent('%(Identity)').Name) - - - - - - - - - - - - diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets new file mode 100644 index 0000000000000..7a084957d5112 --- /dev/null +++ b/eng/testing/tests.wasm.targets @@ -0,0 +1,78 @@ + + + + V8 + $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 + + <_XHarnessArgs>wasm $XHARNESS_COMMAND --app=. --output-directory=$XHARNESS_OUT + + <_XHarnessArgs Condition="'$(Scenario)' != 'WasmTestOnBrowser'">$(_XHarnessArgs) --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js + <_XHarnessArgs Condition="'$(IsFunctionalTest)' == 'true'" >$(_XHarnessArgs) --expected-exit-code=$(ExpectedExitCode) + <_XHarnessArgs Condition="'$(WasmXHarnessArgs)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgs) %24WasmXHarnessArgs + + <_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(Scenario)' != 'BuildWasmApps'">--run WasmTestRunner.dll $(AssemblyName).dll + <_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll --testing + + <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) %24WasmTestAppArgs + + $HARNESS_RUNNER $(_XHarnessArgs) -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) + + + + false + true + false + false + false + + + + + + + PrepareForWasmBuildApp;$(WasmBuildAppDependsOn) + $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'mono', 'wasm', 'emsdk')) + + + + + + + + + + + + + + + + + + + $(BundleDir) + WasmTestRunner.dll + $(MonoProjectRoot)\wasm\runtime-test.js + $(InvariantGlobalization) + true + false + + + + + + $([System.IO.Directory]::GetParent('%(Identity)').Name) + + + + + + + + + + + + From e4312ef5a6821a2946b1ceabaaded2abc56781ff Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 3 Apr 2021 03:11:16 -0400 Subject: [PATCH 3/6] [wasm] set EMSDK_PATH for makefile run-tests- targets --- src/mono/wasm/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mono/wasm/Makefile b/src/mono/wasm/Makefile index 4777abfb104c0..b9b3381f868c2 100644 --- a/src/mono/wasm/Makefile +++ b/src/mono/wasm/Makefile @@ -177,14 +177,14 @@ build-tasks: $(DOTNET) build $(TOP)/src/tasks/WasmBuildTasks $(MSBUILD_ARGS) run-tests-v8-%: - PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=V8 $(MSBUILD_ARGS) + EMSDK_PATH=$(EMSDK_PATH) PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=V8 $(MSBUILD_ARGS) run-tests-sm-%: - PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=SpiderMonkey $(MSBUILD_ARGS) + EMSDK_PATH=$(EMSDK_PATH) PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=SpiderMonkey $(MSBUILD_ARGS) run-tests-jsc-%: - PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=JavaScriptCore $(MSBUILD_ARGS) + EMSDK_PATH=$(EMSDK_PATH) PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) /p:JSEngine=JavaScriptCore $(MSBUILD_ARGS) run-tests-%: - PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS) + EMSDK_PATH=$(EMSDK_PATH) PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/libraries/$*/tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS) run-build-tests: PATH="$(JSVU):$(PATH)" $(DOTNET) build $(TOP)/src/tests/BuildWasmApps/Wasm.Build.Tests/ /t:Test $(_MSBUILD_WASM_BUILD_ARGS) $(MSBUILD_ARGS) From 8c98f299fe861a3a9d2532bd6735ef41aa51fc51 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 3 Apr 2021 21:59:21 -0400 Subject: [PATCH 4/6] [wasm] Update target dependencies for wasm --- eng/testing/tests.mobile.targets | 3 ++- eng/testing/tests.wasm.targets | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index a0e795f8f7c82..efe67cb31648a 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -9,6 +9,7 @@ true true true + BundleTestAppleApp;BundleTestAndroidApp @@ -254,7 +255,7 @@ + DependsOnTargets="Publish;$(BundleTestAppTargets);ArchiveTests" /> diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index 7a084957d5112..6543a871d2b70 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -1,6 +1,8 @@ + $(BundleTestAppTargets);BundleTestWasmApp + V8 $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 From 4242998b933bc7f436d7b01d0c21f10d78ad4b4d Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sun, 4 Apr 2021 01:20:14 -0400 Subject: [PATCH 5/6] Don't overwrite RunScriptCommand, thus fixing wasm build tests --- eng/testing/tests.wasm.targets | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/testing/tests.wasm.targets b/eng/testing/tests.wasm.targets index 6543a871d2b70..505b3a8bde603 100644 --- a/eng/testing/tests.wasm.targets +++ b/eng/testing/tests.wasm.targets @@ -2,7 +2,9 @@ $(BundleTestAppTargets);BundleTestWasmApp + + V8 $(JSEngineArgs) --engine-arg=--stack-trace-limit=1000 From e1af083803457cedb829ea77cc4ee216da141dbf Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sun, 4 Apr 2021 01:23:16 -0400 Subject: [PATCH 6/6] Track property name changes --- .../System.IO.FileSystem.Net5Compat.Tests.csproj | 2 +- .../tests/System.IO.FileSystem.Tests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj index 51daca9429973..1389ee3462649 100644 --- a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj @@ -5,7 +5,7 @@ $(NetCoreAppCurrent)-windows - --working-dir=/test-dir + --working-dir=/test-dir diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index 97b50363adb7c..f22b4500be176 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -4,7 +4,7 @@ true $(NetCoreAppCurrent)-windows;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser - --working-dir=/test-dir + --working-dir=/test-dir