Improving test stability

ladeak · Oct 5, 2023 · 0e46fa0 · 0e46fa0
1 parent 3fe30c0
commit 0e46fa0
Showing 1 changed file with 67 additions and 53 deletions.
diff --git a/tests/CHttp.Tests/Statistics/StatisticsTests.cs b/tests/CHttp.Tests/Statistics/StatisticsTests.cs
@@ -1,66 +1,80 @@
 using System.Diagnostics.Metrics;
 using CHttp.Statitics;
+using Xunit.Sdk;
 
 namespace CHttp.Tests.Statistics;
 
 public class StatisticsTests
 {
-    [Fact]
-    public void BaseicStatistics()
-    {
-        var summaries = new[] {
-            new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(1)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
-            new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(2)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
-            new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(3)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
-        };
+	[Fact]
+	public void BaseicStatistics()
+	{
+		var summaries = new[] {
+			new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(1)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
+			new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(2)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
+			new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(3)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
+		};
 
-        var result = CHttp.Statitics.Statistics.GetStats(new PerformanceMeasurementResults() { Summaries = summaries, TotalBytesRead = 100, Behavior = new(3, 1) });
+		var result = CHttp.Statitics.Statistics.GetStats(new PerformanceMeasurementResults() { Summaries = summaries, TotalBytesRead = 100, Behavior = new(3, 1) });
 
-        Assert.NotNull(result);
-        Assert.Equal(TimeSpan.FromSeconds(2).Ticks, result.Mean);
-        Assert.Equal(8164965.8092772597, result.StdDev);
-        Assert.Equal(4714045.207910317, result.Error, 0.1);
-        Assert.Equal(TimeSpan.FromSeconds(2).Ticks, result.Median);
-        Assert.Equal(TimeSpan.FromSeconds(1).Ticks, result.Min);
-        Assert.Equal(TimeSpan.FromSeconds(3).Ticks, result.Max);
-        Assert.Equal(TimeSpan.FromSeconds(2).Ticks, result.Percentile95th);
-        Assert.Equal(1, result.RequestSec);
-        Assert.Equal(50, result.Throughput);
-    }
+		Assert.NotNull(result);
+		Assert.Equal(TimeSpan.FromSeconds(2).Ticks, result.Mean);
+		Assert.Equal(8164965.8092772597, result.StdDev);
+		Assert.Equal(4714045.207910317, result.Error, 0.1);
+		Assert.Equal(TimeSpan.FromSeconds(2).Ticks, result.Median);
+		Assert.Equal(TimeSpan.FromSeconds(1).Ticks, result.Min);
+		Assert.Equal(TimeSpan.FromSeconds(3).Ticks, result.Max);
+		Assert.Equal(TimeSpan.FromSeconds(2).Ticks, result.Percentile95th);
+		Assert.Equal(1, result.RequestSec);
+		Assert.Equal(50, result.Throughput);
+	}
 
-    // Req/Sec may vary based on parallel tests in the assembly
-    [Theory]
-    [InlineData("Mean", 2000)]
-    [InlineData("StdDev", 816.4965)]
-    [InlineData("Error", 471.4045)]
-    [InlineData("Median", 2000)]
-    [InlineData("Min", 1000)]
-    [InlineData("Max", 3000)]
-    [InlineData("Percentile95", 2000)]
-    [InlineData("Throughput", 0.005)]
-    public async Task TestMetrics(string instrumentName, double expected)
-    {
-        using var listener = new MeterListener();
-        TaskCompletionSource<double> tcs = new TaskCompletionSource<double>();
-        listener.SetMeasurementEventCallback((Instrument instrument, double measurement, ReadOnlySpan<KeyValuePair<string, object?>> tags, object? state) =>
-        {
-            tcs.TrySetResult(measurement);
-        });
-        listener.InstrumentPublished = (instrument, listener) =>
-        {
-            if (instrument.Name == instrumentName)
-                listener.EnableMeasurementEvents(instrument);
-        };
-        listener.Start();
+	[Theory]
+	[InlineData("Mean", 2000)]
+	[InlineData("StdDev", 816.4965)]
+	[InlineData("Error", 471.4045)]
+	[InlineData("Median", 2000)]
+	[InlineData("Min", 1000)]
+	[InlineData("Max", 3000)]
+	[InlineData("Percentile95", 2000)]
+	[InlineData("Throughput", 0.005)]
+	public async Task TestMetrics(string instrumentName, double expected)
+	{
+		// Retry as parallel tests may be also running in this assembly invoking GetStat that produces the metrics.
+		for (int i = 0; i < 3; i++)
+		{
+			using var listener = new MeterListener();
+			TaskCompletionSource<double> tcs = new TaskCompletionSource<double>();
+			listener.SetMeasurementEventCallback((Instrument instrument, double measurement, ReadOnlySpan<KeyValuePair<string, object?>> tags, object? state) =>
+			{
+				tcs.TrySetResult(measurement);
+			});
+			listener.InstrumentPublished = (instrument, listener) =>
+			{
+				if (instrument.Name == instrumentName)
+					listener.EnableMeasurementEvents(instrument);
+			};
+			listener.Start();
 
-        var summaries = new[] {
-            new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(1)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
-            new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(2)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
-            new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(3)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
-        };
+			var summaries = new[]
+			{
+				new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(1)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
+				new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(2)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
+				new Summary("url", new DateTime(2023, 06, 08, 0, 0, 0, DateTimeKind.Utc), TimeSpan.FromSeconds(3)) { ErrorCode = ErrorType.None, HttpStatusCode = 200 },
+			};
 
-        CHttp.Statitics.Statistics.GetStats(new PerformanceMeasurementResults() { Summaries = summaries, TotalBytesRead = 100, Behavior = new(3, 1) });
-        var result = await tcs.Task;
-        Assert.Equal(expected, result, 4);
-    }
+			CHttp.Statitics.Statistics.GetStats(new PerformanceMeasurementResults() { Summaries = summaries, TotalBytesRead = 100, Behavior = new(3, 1) });
+			var result = await tcs.Task;
+			if (Equal(expected, result, 4))
+				return;
+		}
+		Assert.Fail();
+	}
+
+	public static bool Equal(double expected, double actual, int precision)
+	{
+		var expectedRounded = Math.Round(expected, precision);
+		var actualRounded = Math.Round(actual, precision);
+		return Equals(expectedRounded, actualRounded);
+	}
 }