TestModule: add testOnly to select test suites in mill

scalajslib/src/ScalaJSModule.scala

@@ -214,7 +214,9 @@ trait TestScalaJSModule extends ScalaJSModule with TestModule {
 
   override def testLocal(args: String*) = T.command { test(args:_*) }
 
-  override protected def testTask(args: Task[Seq[String]]): Task[(String, Seq[TestRunner.Result])] = T.task {
+  override protected def testTask(args: Task[Seq[String]],
+      globSeletors: Task[Seq[String]]): Task[(String, Seq[TestRunner.Result])] = T.task {
+
     val (close, framework) = mill.scalajslib.ScalaJSWorkerApi.scalaJSWorker().getFramework(
       toolsClasspath().map(_.path),
       jsEnvConfig(),
@@ -228,7 +230,8 @@ trait TestScalaJSModule extends ScalaJSModule with TestModule {
       runClasspath().map(_.path),
       Agg(compile().classes.path),
       args(),
-      T.testReporter
+      T.testReporter,
+      TestRunner.globFilter(globSeletors())
     )
     val res = TestModule.handleResults(doneMsg, results)
     // Hack to try and let the Node.js subprocess finish streaming it's stdout

scalalib/src/TestModule.scala

@@ -56,13 +56,41 @@ trait TestModule extends JavaModule with TaskModule {
     testTask(testCachedArgs)()
   }
 
+  /**
+   * Discovers and runs the module's tests in a subprocess, reporting the
+   * results to the console.
+   * Arguments before "--" will be used as wildcard selector to select
+   * test classes, arguments after "--" will be passed as regular arguments.
+   * `testOnly *foo foobar bar* -- arguments` will test only classes with name
+   * (includes package name) 1. end with "foo", 2. exactly "foobar", 3. start
+   * with "bar", with "arguments" as arguments passing to test framework.
+   */
+  def testOnly(args: String*): Command[(String, Seq[TestRunner.Result])] = {
+    val splitAt = args.indexOf("--")
+    val (selector, testArgs) =
+      if(splitAt == -1) (args, Seq.empty)
+      else {
+        val (s, t) = args.splitAt(splitAt)
+        (s, t.tail)
+      }
+    T.command {
+      testTask(T.task { testArgs }, T.task { selector } )()
+    }
+  }
+
   /** Controls whether the TestRunner should receive it's arguments via an args-file instead of a as long parameter list.
    * Defaults to `true` on Windows, as Windows has a rather short parameter length limit.
    * */
   def testUseArgsFile: T[Boolean] = T { runUseArgsFile() || scala.util.Properties.isWin }
 
+  @deprecated("Use testTask(args, T.task{Seq.empty[String]}) instead.", "mill after 0.9.7")
+  protected def testTask(
+    args: Task[Seq[String]]): Task[(String, Seq[TestRunner.Result])] =
+    testTask(args, T.task{Seq.empty[String]})
+
   protected def testTask(
-      args: Task[Seq[String]]): Task[(String, Seq[TestRunner.Result])] =
+      args: Task[Seq[String]],
+      globSelectors: Task[Seq[String]]): Task[(String, Seq[TestRunner.Result])] =
     T.task {
       val outputPath = T.dest / "out.json"
       val useArgsFile = testUseArgsFile()
@@ -91,7 +119,8 @@ trait TestModule extends JavaModule with TaskModule {
         outputPath = outputPath.toString(),
         colored = T.log.colored,
         testCp = compile().classes.path.toString(),
-        homeStr = T.home.toString()
+        homeStr = T.home.toString(),
+        globSelectors = globSelectors()
       )
 
       val mainArgs = if (useArgsFile) {

scalalib/src/TestRunner.scala

@@ -9,6 +9,8 @@ import mill.scalalib.Lib.discoverTests
 import mill.util.{Ctx, PrintLogger}
 import mill.util.JsonFormatters._
 import sbt.testing._
+
+import java.util.regex.Pattern
 import scala.collection.mutable
 import scala.util.Try
 
@@ -22,7 +24,8 @@ object TestRunner {
       outputPath: String,
       colored: Boolean,
       testCp: String,
-      homeStr: String
+      homeStr: String,
+      globSelectors: Seq[String]
   ) {
     def toArgsSeq: Seq[String] =
       Seq(
@@ -33,6 +36,8 @@ object TestRunner {
           arguments ++
           Seq((sysProps.size * 2).toString) ++
           sysProps.flatMap { case (k, v) => Seq(k, v) } ++
+          Seq(globSelectors.size.toString) ++
+          globSelectors ++
           Seq(outputPath, colored.toString, testCp, homeStr)
       ).flatten
 
@@ -67,7 +72,8 @@ object TestRunner {
         outputPath = outputPath,
         colored = colored,
         testCp = testCp,
-        homeStr = homeStr
+        homeStr = homeStr,
+        globSelectors = Seq.empty
       )
 
     def parseArgs(args: Array[String]): Try[TestArgs] = {
@@ -101,6 +107,7 @@ object TestRunner {
       val classpath = readArray()
       val arguments = readArray()
       val sysProps = readArray()
+      val globFilters = readArray()
       val outputPath = readString()
       val colored = readString()
       val testCp = readString()
@@ -116,7 +123,8 @@ object TestRunner {
         outputPath,
         colored = Seq("true", "1", "on", "yes").contains(colored),
         testCp = testCp,
-        homeStr = homeStr
+        homeStr = homeStr,
+        globFilters
       )
     }
   }
@@ -142,12 +150,15 @@ object TestRunner {
       ctx.log.debug(s"Setting ${testArgs.sysProps.size} system properties")
       testArgs.sysProps.foreach { case (k, v) => System.setProperty(k, v) }
 
+      val filter = globFilter(testArgs.globSelectors)
+
       val result = runTestFramework(
         frameworkInstances = TestRunner.framework(testArgs.framework),
         entireClasspath = Agg.from(testArgs.classpath.map(os.Path(_))),
         testClassfilePath = Agg(os.Path(testArgs.testCp)),
         args = testArgs.arguments,
-        DummyTestReporter
+        DummyTestReporter,
+        filter
       )(ctx)
 
       // Clear interrupted state in case some badly-behaved test suite
@@ -188,6 +199,16 @@ object TestRunner {
                testClassfilePath: Agg[os.Path],
                args: Seq[String],
                testReporter: TestReporter)(implicit ctx: Ctx.Log with Ctx.Home)
+  : (String, Seq[mill.scalalib.TestRunner.Result]) = {
+    runTestFramework(frameworkInstances, entireClasspath, testClassfilePath, args, testReporter, _ => true)
+  }
+
+  def runTestFramework(frameworkInstances: ClassLoader => sbt.testing.Framework,
+               entireClasspath: Agg[os.Path],
+               testClassfilePath: Agg[os.Path],
+               args: Seq[String],
+               testReporter: TestReporter,
+               classFilter: Class[_] => Boolean)(implicit ctx: Ctx.Log with Ctx.Home)
     : (String, Seq[mill.scalalib.TestRunner.Result]) = {
     //Leave the context class loader set and open so that shutdown hooks can access it
     Jvm.inprocess(
@@ -206,7 +227,7 @@ object TestRunner {
           val testClasses = discoverTests(cl, framework, testClassfilePath)
 
           val tasks = runner.tasks(
-            for ((cls, fingerprint) <- testClasses.toArray)
+            for ((cls, fingerprint) <- testClasses.toArray if classFilter(cls))
               yield
                 new TaskDef(
                   cls.getName.stripSuffix("$"),
@@ -285,6 +306,30 @@ object TestRunner {
       .asInstanceOf[sbt.testing.Framework]
   }
 
+  def globFilter(selectors: Seq[String]): Class[_] => Boolean = {
+    val filters = selectors.map{str =>
+      if(str == "*") (_: String) => true
+      else if (str.indexOf('*') == -1) (s: String) => s == str
+      else {
+        val parts = str.split("\\*", -1)
+        parts match {
+          case Array("", suffix) => (s: String) => s.endsWith(suffix)
+          case Array(prefix, "") => (s: String) => s.startsWith(prefix)
+          case _ =>
+            val pattern = Pattern.compile(parts.map(Pattern.quote).mkString(".*"))
+            (s: String) => pattern.matcher(s).matches()
+        }
+      }
+    }
+
+    if(filters.isEmpty) (_: Class[_]) => true
+    else
+      (clz: Class[_]) => {
+        val name = clz.getName.stripSuffix("$")
+        filters.exists(f => f(name))
+      }
+  }
+
   case class Result(fullyQualifiedName: String,
                     selector: String,
                     duration: Long,

scalalib/test/resources/testrunner/test/src/BarTests.scala

@@ -0,0 +1,11 @@
+package mill.scalalib
+
+import utest._
+
+object BarTests extends TestSuite {
+  val tests = Tests{
+    test("test"){
+      true
+    }
+  }
+}

scalalib/test/resources/testrunner/test/src/FooTests.scala

@@ -0,0 +1,11 @@
+package mill.scalalib
+
+import utest._
+
+object FooTests extends TestSuite {
+  val tests = Tests{
+    test("test"){
+      true
+    }
+  }
+}

scalalib/test/resources/testrunner/test/src/FoobarTests.scala

@@ -0,0 +1,11 @@
+package mill.scalalib
+
+import utest._
+
+object FoobarTests extends TestSuite {
+  val tests = Tests{
+    test("test"){
+      true
+    }
+  }
+}

scalalib/test/src/TestRunnerTests.scala

@@ -1,37 +1,102 @@
 package mill.scalalib
 
-import scala.util.Success
+import mill.{Agg, T}
 
+import scala.util.Success
 import mill.scalalib.TestRunner.TestArgs
+import mill.util.{TestEvaluator, TestUtil}
 import org.scalacheck.Prop.forAll
 import utest._
+import utest.framework.TestPath
 
 object TestRunnerTests extends TestSuite {
+  object testrunner extends TestUtil.BaseModule with ScalaModule{
+    override def millSourcePath =  TestUtil.getSrcPathBase() / millOuterCtx.enclosing.split('.')
+
+    def scalaVersion = "2.12.4"
+
+    object test extends super.Tests with TestModule.Utest {
+
+      override def ivyDeps = T{
+        super.ivyDeps() ++ Agg(ivy"com.lihaoyi::utest:0.7.10")
+      }
+    }
+  }
+
+  val resourcePath = os.pwd / 'scalalib / 'test / 'resources / 'testrunner
+
+  def workspaceTest[T](
+      m: TestUtil.BaseModule,
+      resourcePath: os.Path = resourcePath)(t: TestEvaluator => T)(
+      implicit tp: TestPath): T = {
+    val eval = new TestEvaluator(m)
+    os.remove.all(m.millSourcePath)
+    os.remove.all(eval.outPath)
+    os.makeDir.all(m.millSourcePath / os.up)
+    os.copy(resourcePath, m.millSourcePath)
+    t(eval)
+  }
+
   override def tests: Tests = Tests {
-    test("TestArgs args serialization") {
-      forAll {
-        (
-            frameworks: Seq[String],
-            classpath: Seq[String],
-            arguments: Seq[String],
-            sysProps: Map[String, String],
-            outputPath: String,
-            colored: Boolean,
-            testCp: String,
-            homeStr: String
-        ) =>
-          val testArgs = TestArgs(
-            frameworks,
-            classpath,
-            arguments,
-            sysProps,
-            outputPath,
-            colored,
-            testCp,
-            homeStr
+    "TestArgs" - {
+      test("args serialization") {
+        forAll { (globSelectors: Seq[String]) =>
+          forAll {
+            (
+              framework: String,
+              classpath: Seq[String],
+              arguments: Seq[String],
+              sysProps: Map[String, String],
+              outputPath: String,
+              colored: Boolean,
+              testCp: String,
+              homeStr: String
+            ) =>
+              val testArgs = TestArgs(
+                framework,
+                classpath,
+                arguments,
+                sysProps,
+                outputPath,
+                colored,
+                testCp,
+                homeStr,
+                globSelectors
+              )
+              TestArgs.parseArgs(testArgs.toArgsSeq.toArray) == Success(testArgs)
+          }
+        }.check
+      }
+    }
+    "TestRunner" - {
+      "test case lookup" - workspaceTest(testrunner) { eval =>
+        val Right((result, _)) = eval.apply(testrunner.test.test())
+        val test = result.asInstanceOf[(String, Seq[mill.scalalib.TestRunner.Result])]
+        assert(
+          test._2.size == 3
+        )
+      }
+      "testOnly" - {
+        def testOnly(eval: TestEvaluator, args: Seq[String], size: Int) = {
+          val Right((result1, _)) = eval.apply(testrunner.test.testOnly(args:_*))
+          val testOnly = result1.asInstanceOf[(String, Seq[mill.scalalib.TestRunner.Result])]
+          assert(
+            testOnly._2.size == size
           )
-          TestArgs.parseArgs(testArgs.toArgsSeq.toArray) == Success(testArgs)
-      }.check
+        }
+        "suffix" - workspaceTest(testrunner) { eval =>
+          testOnly(eval, Seq("*arTests"), 2)
+        }
+        "prefix" - workspaceTest(testrunner) { eval =>
+          testOnly(eval, Seq("mill.scalalib.FooT*"), 1)
+        }
+        "exactly" - workspaceTest(testrunner) { eval =>
+          testOnly(eval, Seq("mill.scalalib.FooTests"), 1)
+        }
+        "multi" - workspaceTest(testrunner) { eval =>
+          testOnly(eval, Seq("*Bar*", "*bar*"), 2)
+        }
+      }
     }
   }
 }

scalanativelib/src/ScalaNativeModule.scala

@@ -176,7 +176,8 @@ trait ScalaNativeModule extends ScalaModule { outer =>
 
 trait TestScalaNativeModule extends ScalaNativeModule with TestModule {
   override def testLocal(args: String*) = T.command { test(args:_*) }
-  override protected def testTask(args: Task[Seq[String]]): Task[(String, Seq[TestRunner.Result])] = T.task {
+  override protected def testTask(args: Task[Seq[String]],
+      globSeletors: Task[Seq[String]]): Task[(String, Seq[TestRunner.Result])] = T.task {
 
     val getFrameworkResult = scalaNativeWorker().getFramework(
       nativeLink().toIO,
@@ -192,7 +193,8 @@ trait TestScalaNativeModule extends ScalaNativeModule with TestModule {
       runClasspath().map(_.path),
       Agg(compile().classes.path),
       args(),
-      T.testReporter
+      T.testReporter,
+      TestRunner.globFilter(globSeletors())
     )
     val res = TestModule.handleResults(doneMsg, results)
     // Hack to try and let the Scala Native subprocess finish streaming it's stdout