Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wasm] Add support for a random test case orderer, for xunit tests #65628

Merged
merged 8 commits into from
Feb 21, 2022
Next Next commit
[wasm] Add support for a random test case orderer, for xunit tests
This is enabled by default for wasm with
`$(XUnitUseRandomizedTestOrderer)=true`.

When the library tests run, they print two messages like:

```
  info: Using random seed for test cases: 700149826
  info: Using random seed for collections: 700149826
  info: Starting:    System.Collections.Immutable.Tests.dll
```

These seeds are picked randomly every time the tests are run. To run the
tests with a specific seed, use environment variable
`XUNIT_RANDOM_ORDER_SEED`.

When running with tests, this can be used as:

    `WasmXHarnessMonoArgs="--setenv=XUNIT_RANDOM_ORDER_SEED=<seed>`
radical authored Feb 20, 2022
commit ce4eef991a0bf81a6bcd050f5aea4fae6927f0ac
4 changes: 4 additions & 0 deletions eng/testing/tests.mobile.targets
Original file line number Diff line number Diff line change
@@ -51,6 +51,10 @@
<AdditionalXHarnessArguments Condition="'$(XUnitClassName)' != ''">$(AdditionalXHarnessArguments) -- -c=$(XUnitClassName)</AdditionalXHarnessArguments>
</PropertyGroup>

<ItemGroup Condition="'$(XUnitUseRandomizedTestOrderer)' == 'true'">
<Compile Include="$(RepoRoot)src\libraries\Common\tests\Tests\RandomizedTestOrderAssemblyInfo.cs" />
</ItemGroup>

<UsingTask Condition="'$(RunAOTCompilation)' == 'true'" TaskName="MonoAOTCompiler" AssemblyFile="$(MonoAOTCompilerTasksAssemblyPath)" />
<Import Condition="'$(RunAOTCompilation)' == 'true'" Project="$(MonoAOTCompilerDir)MonoAOTCompiler.props" />

1 change: 1 addition & 0 deletions eng/testing/tests.wasm.targets
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@
<_ShellCommandSeparator Condition="'$(OS)' != 'Windows_NT'">&amp;&amp;</_ShellCommandSeparator>
<WasmNativeStrip>false</WasmNativeStrip>
<_WasmMainJSFileName Condition="'$(WasmMainJSPath)' != ''">$([System.IO.Path]::GetFileName('$(WasmMainJSPath)'))</_WasmMainJSFileName>
<XUnitUseRandomizedTestOrderer>true</XUnitUseRandomizedTestOrderer>
</PropertyGroup>

<PropertyGroup>
77 changes: 77 additions & 0 deletions src/libraries/Common/tests/TestUtilities/RandomTestCaseOrderer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading;
using Xunit.Abstractions;
using Xunit.Sdk;

#nullable enable

namespace TestUtilities;

// Based on https://github.com/xunit/xunit/blob/v2/src/xunit.execution/Sdk/DefaultTestCaseOrderer.cs

public class RandomTestCaseOrderer : ITestCaseOrderer
{
public const string RandomSeedEnvironmentVariableName = "XUNIT_RANDOM_ORDER_SEED";

public static readonly Lazy<int> LazySeed = new (GetSeed, LazyThreadSafetyMode.ExecutionAndPublication);
private readonly IMessageSink _diagnosticMessageSink;

private static int GetSeed()
{
string? seedEnvVar = Environment.GetEnvironmentVariable(RandomSeedEnvironmentVariableName);
if (string.IsNullOrEmpty(seedEnvVar) || !int.TryParse(seedEnvVar, out int seed))
{
seed = new Random().Next();
}

return seed;
}

public RandomTestCaseOrderer(IMessageSink diagnosticMessageSink)
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Using random seed for test cases: {LazySeed.Value}"));
_diagnosticMessageSink = diagnosticMessageSink;
}

public IEnumerable<TTestCase> OrderTestCases<TTestCase>(IEnumerable<TTestCase> testCases) where TTestCase : ITestCase
=> TryRandomize(testCases.ToList(), _diagnosticMessageSink, out List<TTestCase>? randomizedTests)
? randomizedTests
: testCases;

public static bool TryRandomize<T>(List<T> tests, IMessageSink messageSink, [NotNullWhen(true)] out List<T>? randomizedTests)
{
randomizedTests = null;
try
{
randomizedTests = Randomize(tests.ToList());
return true;
}
catch (Exception ex)
{
messageSink.OnMessage(new DiagnosticMessage($"Failed to randomize test cases: {ex}"));
return false;
}

static List<T> Randomize(List<T> tests)
{
var result = new List<T>(tests.Count);

var randomizer = new Random(LazySeed.Value);

while (tests.Count > 0)
{
int next = randomizer.Next(tests.Count);
result.Add(tests[next]);
tests.RemoveAt(next);
}

return result;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Generic;
using System.Linq;
using Xunit;
using Xunit.Abstractions;
using Xunit.Sdk;

#nullable enable

namespace TestUtilities;

public class RandomTestCollectionOrderer : ITestCollectionOrderer
{
private readonly IMessageSink _diagnosticMessageSink;

public RandomTestCollectionOrderer(IMessageSink diagnosticMessageSink)
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage(
$"Using random seed for collections: {RandomTestCaseOrderer.LazySeed.Value}"));
_diagnosticMessageSink = diagnosticMessageSink;
}

public IEnumerable<ITestCollection> OrderTestCollections(IEnumerable<ITestCollection> testCollections)
=> RandomTestCaseOrderer.TryRandomize(testCollections.ToList(), _diagnosticMessageSink, out List<ITestCollection>? randomizedTests)
? randomizedTests
: testCollections;
}
3 changes: 3 additions & 0 deletions src/libraries/Common/tests/TestUtilities/TestUtilities.csproj
Original file line number Diff line number Diff line change
@@ -43,6 +43,9 @@
variant from the Common folder and adding the missing members manually.
-->
<Compile Include="Interop\Interop.Libraries.cs" />

<Compile Include="RandomTestCaseOrderer.cs" />
<Compile Include="RandomTestCollectionOrderer.cs" />
</ItemGroup>
<!-- Windows imports -->
<ItemGroup>
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Xunit;

[assembly: TestCaseOrderer("TestUtilities.RandomTestCaseOrderer", "TestUtilities")]
[assembly: TestCollectionOrderer("TestUtilities.RandomTestCollectionOrderer", "TestUtilities")]