From 888bc79f793a74f22d3453351c7c722b892549fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93lafur=20P=C3=A1ll=20Geirsson?= Date: Thu, 6 Jan 2022 19:01:43 +0100 Subject: [PATCH 1/2] Make flaky tag work with Scalacheck suites Previously, the Scalacheck suite transform was registered after the `.flaky` transform so a flaky test would fail even if it was OK to ignore flaky failures. Reordering the transforms fixes the issue. --- .../shared/src/main/scala/munit/ScalaCheckSuite.scala | 2 +- .../src/main/scala/munit/ScalaCheckFrameworkSuite.scala | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/munit-scalacheck/shared/src/main/scala/munit/ScalaCheckSuite.scala b/munit-scalacheck/shared/src/main/scala/munit/ScalaCheckSuite.scala index f86ca616..bb5b61d2 100644 --- a/munit-scalacheck/shared/src/main/scala/munit/ScalaCheckSuite.scala +++ b/munit-scalacheck/shared/src/main/scala/munit/ScalaCheckSuite.scala @@ -29,7 +29,7 @@ trait ScalaCheckSuite extends FunSuite { implicit def unitToProp(unit: Unit): Prop = Prop.passed override def munitTestTransforms: List[TestTransform] = - super.munitTestTransforms :+ scalaCheckPropTransform + scalaCheckPropTransform +: super.munitTestTransforms protected def scalaCheckTestParameters = ScalaCheckTest.Parameters.default diff --git a/tests/shared/src/main/scala/munit/ScalaCheckFrameworkSuite.scala b/tests/shared/src/main/scala/munit/ScalaCheckFrameworkSuite.scala index 9d6b22e1..c03e18a1 100644 --- a/tests/shared/src/main/scala/munit/ScalaCheckFrameworkSuite.scala +++ b/tests/shared/src/main/scala/munit/ScalaCheckFrameworkSuite.scala @@ -38,6 +38,13 @@ class ScalaCheckFrameworkSuite extends ScalaCheckSuite { } } + override def munitFlakyOK: Boolean = true + test("flaky test".flaky) { + forAll { (i: Int) => + assertEquals(1, 0) + } + } + } object ScalaCheckFrameworkSuite @@ -78,5 +85,6 @@ object ScalaCheckFrameworkSuite |Falsified after 0 passed tests. |> ARG_0: -1 |> ARG_0_ORIGINAL: 2147483647 + |==> skipped munit.ScalaCheckFrameworkSuite.flaky test - ignoring flaky test failure |""".stripMargin ) From 1a48ba10ecffc7c447e545484978570f354e348e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93lafur=20P=C3=A1ll=20Geirsson?= Date: Mon, 17 Jan 2022 09:23:36 +0100 Subject: [PATCH 2/2] Fix duplicate reporting of ignored tests in JS/Native There was a bug in the JS/Native implementation of `RunNotifier` so that it would report that a test was both ignored and that it succeeded. --- .../shared/src/main/scala/munit/ScalaCheckSuite.scala | 1 - .../munit/internal/junitinterface/MUnitRunNotifier.scala | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/munit-scalacheck/shared/src/main/scala/munit/ScalaCheckSuite.scala b/munit-scalacheck/shared/src/main/scala/munit/ScalaCheckSuite.scala index bb5b61d2..30a94856 100644 --- a/munit-scalacheck/shared/src/main/scala/munit/ScalaCheckSuite.scala +++ b/munit-scalacheck/shared/src/main/scala/munit/ScalaCheckSuite.scala @@ -10,7 +10,6 @@ import scala.util.Try import munit.internal.FutureCompat._ trait ScalaCheckSuite extends FunSuite { - def property( name: String )(body: => Prop)(implicit loc: Location): Unit = { diff --git a/munit/non-jvm/src/main/scala/munit/internal/junitinterface/MUnitRunNotifier.scala b/munit/non-jvm/src/main/scala/munit/internal/junitinterface/MUnitRunNotifier.scala index e70b0d6f..b03af1ff 100644 --- a/munit/non-jvm/src/main/scala/munit/internal/junitinterface/MUnitRunNotifier.scala +++ b/munit/non-jvm/src/main/scala/munit/internal/junitinterface/MUnitRunNotifier.scala @@ -9,7 +9,7 @@ class MUnitRunNotifier(reporter: JUnitReporter) extends RunNotifier { var ignored = 0 var total = 0 var startedTimestamp = 0L - val isFailed: mutable.Set[String] = mutable.Set.empty[String] + val isReported: mutable.Set[Description] = mutable.Set.empty[Description] override def fireTestSuiteStarted(description: Description): Unit = { reporter.reportTestSuiteStarted() } @@ -23,11 +23,13 @@ class MUnitRunNotifier(reporter: JUnitReporter) extends RunNotifier { } override def fireTestIgnored(description: Description): Unit = { ignored += 1 + isReported += description reporter.reportTestIgnored(description.getMethodName) } override def fireTestAssumptionFailed( failure: notification.Failure ): Unit = { + isReported += failure.description reporter.reportAssumptionViolation( failure.description.getMethodName, elapsedMillis(), @@ -36,7 +38,7 @@ class MUnitRunNotifier(reporter: JUnitReporter) extends RunNotifier { } override def fireTestFailure(failure: notification.Failure): Unit = { val methodName = failure.description.getMethodName - isFailed += methodName + isReported += failure.description reporter.reportTestFailed( methodName, failure.ex, @@ -46,7 +48,7 @@ class MUnitRunNotifier(reporter: JUnitReporter) extends RunNotifier { override def fireTestFinished(description: Description): Unit = { val methodName = description.getMethodName total += 1 - if (!isFailed(methodName)) { + if (!isReported(description)) { reporter.reportTestPassed( methodName, elapsedMillis()