Skip to content

Commit

Permalink
Refactoring xUnit tests to avoid non-serializable objects in MemberData
Browse files Browse the repository at this point in the history 
Inspired by aspnet/KestrelHttpServer#2520
  • Loading branch information
@AndreyAkinshin
AndreyAkinshin committed Dec 8, 2018
1 parent 91e16a9 commit 89255c9
Show file tree
Hide file tree
Showing 5 changed files with 114 additions and 66 deletions.
72 changes: 39 additions & 33 deletions tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Running;
using JetBrains.Annotations;
using Xunit;

namespace BenchmarkDotNet.Tests.Engine
Expand All @@ -14,7 +15,7 @@ public class EngineFactoryTests
{
int timesBenchmarkCalled = 0, timesOverheadCalled = 0;
int timesGlobalSetupCalled = 0, timesGlobalCleanupCalled = 0, timesIterationSetupCalled = 0, timesIterationCleanupCalled = 0;

TimeSpan IterationTime => TimeSpan.FromMilliseconds(EngineResolver.Instance.Resolve(Job.Default, RunMode.IterationTimeCharacteristic).ToMilliseconds());

IResolver DefaultResolver => BenchmarkRunner.DefaultResolver;
Expand All @@ -23,32 +24,36 @@ public class EngineFactoryTests
void IterationSetup() => timesIterationSetupCalled++;
void IterationCleanup() => timesIterationCleanupCalled++;
void GlobalCleanup() => timesGlobalCleanupCalled++;

void Throwing(long _) => throw new InvalidOperationException("must NOT be called");

void VeryTimeConsumingSingle(long _)
{
timesBenchmarkCalled++;
Thread.Sleep(IterationTime);
}
void InstantNoUnroll(long invocationCount) => timesBenchmarkCalled += (int)invocationCount;

void InstantNoUnroll(long invocationCount) => timesBenchmarkCalled += (int) invocationCount;
void InstantUnroll(long _) => timesBenchmarkCalled += 16;
void OverheadNoUnroll(long invocationCount) => timesOverheadCalled += (int)invocationCount;

void OverheadNoUnroll(long invocationCount) => timesOverheadCalled += (int) invocationCount;
void OverheadUnroll(long _) => timesOverheadCalled += 16;

public static IEnumerable<object[]> JobsWhichDontRequireJitting()
private static readonly Dictionary<string, Job> JobsWhichDontRequireJitting = new Dictionary<string, Job>
{
yield return new object[]{ Job.Dry };
yield return new object[]{ Job.Default.With(RunStrategy.ColdStart) };
yield return new object[]{ Job.Default.With(RunStrategy.Monitoring) };
}
{ "Dry", Job.Dry },
{ "ColdStart", Job.Default.With(RunStrategy.ColdStart) },
{ "Monitoring", Job.Default.With(RunStrategy.Monitoring) }
};

[UsedImplicitly]
public static TheoryData<string> JobsWhichDontRequireJittingNames => TheoryDataHelper.Create(JobsWhichDontRequireJitting.Keys);

[Theory]
[MemberData(nameof(JobsWhichDontRequireJitting))]
public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled(Job job)
[MemberData(nameof(JobsWhichDontRequireJittingNames))]
public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled(string jobName)
{
var job = JobsWhichDontRequireJitting[jobName];
var engineParameters = CreateEngineParameters(mainNoUnroll: Throwing, mainUnroll: Throwing, job: job);

var engine = new EngineFactory().CreateReadyToRun(engineParameters);
Expand All @@ -58,7 +63,7 @@ public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled(Job job)
Assert.Equal(0, timesBenchmarkCalled);
Assert.Equal(0, timesOverheadCalled);
Assert.Equal(0, timesIterationCleanupCalled);
Assert.Equal(0, timesGlobalCleanupCalled);
Assert.Equal(0, timesGlobalCleanupCalled);

engine.Dispose();

Expand All @@ -81,7 +86,7 @@ public void ForDefaultSettingsVeryTimeConsumingBenchmarksAreExecutedOncePerItera

Assert.Equal(1, engine.TargetJob.Run.InvocationCount); // call the benchmark once per iteration
Assert.Equal(1, engine.TargetJob.Run.UnrollFactor); // no unroll factor

Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way
Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case

Expand All @@ -95,7 +100,7 @@ public void ForJobsWithExplicitUnrollFactorTheGlobalSetupIsCalledAndMultiActionC
=> AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job.Default.WithUnrollFactor(16));

[Fact]
public void ForJobsThatDontRequirePilotTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted()
public void ForJobsThatDontRequirePilotTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted()
=> AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job.Default.WithInvocationCount(100));

private void AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job job)
Expand All @@ -109,15 +114,15 @@ private void AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job job)
Assert.Equal(16, timesBenchmarkCalled);
Assert.Equal(16, timesOverheadCalled);
Assert.Equal(1, timesIterationCleanupCalled);
Assert.Equal(0, timesGlobalCleanupCalled);
Assert.Equal(0, timesGlobalCleanupCalled);

Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched

engine.Dispose();

Assert.Equal(1, timesGlobalCleanupCalled);
}

[Fact]
public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWithUnrollFactorForDefaultSettings()
{
Expand All @@ -126,12 +131,12 @@ public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWit
var engine = new EngineFactory().CreateReadyToRun(engineParameters);

Assert.Equal(1, timesGlobalSetupCalled);
Assert.Equal(1+1, timesIterationSetupCalled); // once for single and & once for 16
Assert.Equal(1 + 1, timesIterationSetupCalled); // once for single and & once for 16
Assert.Equal(1 + 16, timesBenchmarkCalled);
Assert.Equal(1 + 16, timesOverheadCalled);
Assert.Equal(1+1, timesIterationCleanupCalled); // once for single and & once for 16
Assert.Equal(1 + 1, timesIterationCleanupCalled); // once for single and & once for 16
Assert.Equal(0, timesGlobalCleanupCalled);

Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched

Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic));
Expand All @@ -147,9 +152,9 @@ public void MediumTimeConsumingBenchmarksShouldStartPilotFrom2AndIncrementItWith
var unrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver);

const int times = 5; // how many times we should invoke the benchmark per iteration

var mediumTime = TimeSpan.FromMilliseconds(IterationTime.TotalMilliseconds / times);

void MediumNoUnroll(long invocationCount)
{
for (int i = 0; i < invocationCount; i++)
Expand All @@ -159,17 +164,17 @@ void MediumNoUnroll(long invocationCount)
Thread.Sleep(mediumTime);
}
}

void MediumUnroll(long _)
{
timesBenchmarkCalled += unrollFactor;

for (int i = 0; i < unrollFactor; i++) // the real unroll factor obviously does not use loop ;)
Thread.Sleep(mediumTime);
}

var engineParameters = CreateEngineParameters(mainNoUnroll: MediumNoUnroll, mainUnroll: MediumUnroll, job: Job.Default);

var engine = new EngineFactory().CreateReadyToRun(engineParameters);

Assert.Equal(1, timesGlobalSetupCalled);
Expand All @@ -178,11 +183,12 @@ void MediumUnroll(long _)
Assert.Equal(1, timesOverheadCalled);
Assert.Equal(1, timesIterationCleanupCalled);
Assert.Equal(0, timesGlobalCleanupCalled);

Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic)); // we need to run the pilot!
Assert.Equal(1, engine.TargetJob.Run.UnrollFactor); // no unroll factor!
Assert.Equal(2, engine.TargetJob.Accuracy.MinInvokeCount); // we start from two (we know that 1 is not enough, the default is 4 so we need to override it)

Assert.Equal(1, engine.TargetJob.Run.UnrollFactor); // no unroll factor!
Assert.Equal(2,
engine.TargetJob.Accuracy.MinInvokeCount); // we start from two (we know that 1 is not enough, the default is 4 so we need to override it)

Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way
Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case

Expand Down
21 changes: 13 additions & 8 deletions tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,18 @@
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Portability;
using BenchmarkDotNet.Tests.Builders;
using JetBrains.Annotations;
using Xunit;

namespace BenchmarkDotNet.Tests.Environments
{
public class HostEnvironmentInfoTests
{
[Theory]
[MemberData(nameof(Hypervisors))]
public void ReturnsHipervisorNameWhenItsDetected(VirtualMachineHypervisor hypervisor)
[MemberData(nameof(HypervisorNames))]
public void ReturnsHypervisorNameWhenItsDetected(string hypervisorName)
{
var hypervisor = Hypervisors[hypervisorName];
var info = new HostEnvironmentInfoBuilder()
.WithVMHypervisor(hypervisor)
.Build();
Expand All @@ -24,12 +26,15 @@ public void ReturnsHipervisorNameWhenItsDetected(VirtualMachineHypervisor hyperv
Assert.Equal(expected, line);
}

public static IEnumerable<object[]> Hypervisors()
private static readonly IDictionary<string, VirtualMachineHypervisor> Hypervisors = new Dictionary<string, VirtualMachineHypervisor>
{
yield return new object[] { HyperV.Default };
yield return new object[] { VirtualBox.Default };
yield return new object[] { VMware.Default };
}
{ HyperV.Default.Name, HyperV.Default },
{ VirtualBox.Default.Name, VirtualBox.Default },
{ VMware.Default.Name, VMware.Default }
};

[UsedImplicitly]
public static TheoryData<string> HypervisorNames => TheoryDataHelper.Create(Hypervisors.Keys);

[Fact]
public void DoesntReturnHypervisorNameWhenItsNotDetected()
Expand All @@ -45,4 +50,4 @@ public void DoesntReturnHypervisorNameWhenItsNotDetected()
Assert.Equal(expected, line);
}
}
}
}
28 changes: 12 additions & 16 deletions tests/BenchmarkDotNet.Tests/Exporters/CommonExporterApprovalTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
using BenchmarkDotNet.Tests.Mocks;
using JetBrains.Annotations;
using Xunit;

namespace BenchmarkDotNet.Tests.Exporters
{
// In case of failed approval tests, use the following reporter:
Expand All @@ -31,27 +32,22 @@ public CommonExporterApprovalTests()
initCulture = Thread.CurrentThread.CurrentCulture;
}

[UsedImplicitly]
public static TheoryData<CultureInfo> GetCultureInfos()
private static readonly Dictionary<string, CultureInfo> CultureInfos = new Dictionary<string, CultureInfo>
{
var cultures = new List<CultureInfo>
{
CultureInfo.InvariantCulture,
new CultureInfo("ru-RU"),
new CultureInfo("en-US")
};
{ "", CultureInfo.InvariantCulture },
{ "ru-RU", new CultureInfo("ru-RU") },
{ "en-US", new CultureInfo("en-US") }
};

var theoryData = new TheoryData<CultureInfo>();
foreach (var cultureInfo in cultures)
theoryData.Add(cultureInfo);
return theoryData;
}
[UsedImplicitly]
public static TheoryData<string> CultureInfoNames => TheoryDataHelper.Create(CultureInfos.Keys);

[Theory]
[MethodImpl(MethodImplOptions.NoInlining)] // required by the Approval test framework, do NOT remove
[MemberData(nameof(GetCultureInfos))]
public void Exporters(CultureInfo cultureInfo)
[MemberData(nameof(CultureInfoNames))]
public void Exporters(string cultureInfoName)
{
var cultureInfo = CultureInfos[cultureInfoName];
NamerFactory.AdditionalInformation = $"{GetName(cultureInfo)}";
Thread.CurrentThread.CurrentCulture = cultureInfo;

Expand Down Expand Up @@ -112,4 +108,4 @@ public void Dispose()
Thread.CurrentThread.CurrentCulture = initCulture;
}
}
}
}
43 changes: 34 additions & 9 deletions tests/BenchmarkDotNet.Tests/Exporters/XmlSerializerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Globalization;
using System.Text;
using BenchmarkDotNet.Exporters.Xml;
using JetBrains.Annotations;
using Xunit;

namespace BenchmarkDotNet.Tests.Exporters
Expand Down Expand Up @@ -134,20 +135,44 @@ public void WithExcludedPropertyThrowsGivenNameIsNullOrWhiteSpace(string name, T
}

[Theory]
[MemberData(nameof(SerializeTestData))]
public void SerializeThrowsGivenNullArguments(MockXmlWriter writer, object source, Type exception)
[MemberData(nameof(SerializeTestDataNames))]
public void SerializeThrowsGivenNullArguments(string dataName)
{
IXmlSerializer serializer = XmlSerializer.GetBuilder(typeof(MockSource)).Build();
var (writer, source, exception) = SerializeTestDataItems[dataName];
var serializer = XmlSerializer.GetBuilder(typeof(MockSource)).Build();
Assert.Throws(exception, () => serializer.Serialize(writer, source));
}

public static IEnumerable<object[]> SerializeTestData { get; } =
new List<object[]>
private class SerializeTestData
{
private MockXmlWriter Writer { get; }
private object Source { get; }
private Type Exception { get; }

public SerializeTestData(MockXmlWriter writer, object source, Type exception)
{
Writer = writer;
Source = source;
Exception = exception;
}

public void Deconstruct(out MockXmlWriter writer, out object source, out Type exception)
{
new object[] { null, null, typeof(ArgumentNullException) },
new object[] { null, new MockSource(), typeof(ArgumentNullException) },
new object[] { new MockXmlWriter(), null, typeof(ArgumentNullException) }
};
writer = Writer;
source = Source;
exception = Exception;
}
}

private static readonly Dictionary<string, SerializeTestData> SerializeTestDataItems = new Dictionary<string, SerializeTestData>
{
{"Null", new SerializeTestData(null, null, typeof(ArgumentNullException))},
{"MockSource", new SerializeTestData(null, new MockSource(), typeof(ArgumentNullException))},
{"MockXmlWriter", new SerializeTestData(new MockXmlWriter(), null, typeof(ArgumentNullException))}
};

[UsedImplicitly]
public static TheoryData<string> SerializeTestDataNames => TheoryDataHelper.Create(SerializeTestDataItems.Keys);

[Fact]
public void WritesElementStringGivenSimpleCollectionItem()
Expand Down
16 changes: 16 additions & 0 deletions tests/BenchmarkDotNet.Tests/TheoryDataHelper.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
using System.Collections.Generic;
using Xunit;

namespace BenchmarkDotNet.Tests
{
public static class TheoryDataHelper
{
public static TheoryData<string> Create(IEnumerable<string> values)
{
var data = new TheoryData<string>();
foreach (string value in values)
data.Add(value);
return data;
}
}
}

0 comments on commit 89255c9

Please sign in to comment.