Add native image test task (#60)

* add NativeImageTest and nativeImageTestRun

* add nativeImageTestRunAgent

* add basic-test

* fix imports in basic-test

* add cross-build-test

* fix nativeImageTestRunAgent

* add agent-test

* remove new line from test output

* nativeImageTestAgentOutputDir same as nativeImageAgentOutputDir by default

* add nativeImageTestRunOptions

* log command

plugin/src/main/scala/sbtnativeimage/NativeImagePlugin.scala

@@ -24,12 +24,18 @@ object NativeImagePlugin extends AutoPlugin {
 
   object autoImport {
     val NativeImage: Configuration = config("native-image")
+    val NativeImageTest: Configuration = config("native-image-test")
     val NativeImageInternal: Configuration =
       config("native-image-internal").hide
+    val NativeImageTestInternal: Configuration =
+      config("native-image-test-internal").hide
 
     lazy val nativeImageReady: TaskKey[() => Unit] = taskKey[() => Unit](
       "This function is called when the native image is ready."
     )
+    lazy val nativeImageTestReady: TaskKey[() => Unit] = taskKey[() => Unit](
+      "This function is called when the native image of tests is ready."
+    )
     lazy val nativeImageVersion: SettingKey[String] = settingKey[String](
       "The version of GraalVM to use by default."
     )
@@ -51,31 +57,59 @@ object NativeImagePlugin extends AutoPlugin {
     lazy val nativeImageCommand: TaskKey[Seq[String]] = taskKey[Seq[String]](
       "The command arguments to launch the GraalVM native-image binary."
     )
+    lazy val nativeImageTestCommand: TaskKey[Seq[String]] = taskKey[Seq[String]](
+      "The command arguments to launch the GraalVM native-image binary of tests."
+    )
     lazy val nativeImageRunAgent: InputKey[Unit] = inputKey[Unit](
       "Run application, tracking all usages of dynamic features of an execution with `native-image-agent`."
     )
+    lazy val nativeImageTestRunAgent: InputKey[Unit] = inputKey[Unit](
+      "Run tests, tracking all usages of dynamic features of an execution with `native-image-agent`."
+    )
     lazy val nativeImageAgentOutputDir: SettingKey[File] = settingKey[File](
       "Directory where `native-image-agent` should put generated configurations."
     )
+    lazy val nativeImageTestAgentOutputDir: SettingKey[File] = settingKey[File](
+      "Directory where `native-image-agent` should put generated configurations for tests."
+    )
     lazy val nativeImageAgentMerge: SettingKey[Boolean] = settingKey[Boolean](
       "Whether `native-image-agent` should merge generated configurations." +
         s" (See $assistedConfigurationOfNativeImageBuildsLink for details)"
     )
+    lazy val nativeImageTestAgentMerge: SettingKey[Boolean] = settingKey[Boolean](
+      "Whether `native-image-agent` should merge generated configurations for tests." +
+        s" (See $assistedConfigurationOfNativeImageBuildsLink for details)"
+    )
     lazy val nativeImage: TaskKey[File] = taskKey[File](
       "Generate a native image for this project."
     )
+    lazy val nativeImageTest: TaskKey[File] = taskKey[File](
+      "Generate a native image for tests of this project."
+    )
     lazy val nativeImageRun: InputKey[Unit] = inputKey[Unit](
       "Run the generated native-image binary without linking."
     )
+    lazy val nativeImageTestRun: InputKey[Unit] = inputKey[Unit](
+      "Run the generated native-image binary for tests without linking."
+    )
+    lazy val nativeImageTestRunOptions: TaskKey[Seq[String]] = taskKey[Seq[String]](
+      "Extra command-line arguments that should be forwarded to the tests."
+    )
     lazy val nativeImageCopy: InputKey[Unit] = inputKey[Unit](
       "Link the native image and copy the resulting binary to the provided file argument."
     )
     lazy val nativeImageOutput: SettingKey[File] = settingKey[File](
       "The binary that is produced by native-image"
     )
+    lazy val nativeImageTestOutput: SettingKey[File] = settingKey[File](
+      "The binary that is produced by tests native-image"
+    )
     lazy val nativeImageOptions: TaskKey[Seq[String]] = taskKey[Seq[String]](
       "Extra command-line arguments that should be forwarded to the native-image optimizer."
     )
+    lazy val nativeImageTestOptions: TaskKey[Seq[String]] = taskKey[Seq[String]](
+      "Extra command-line arguments that should be forwarded to the native-image optimizer."
+    )
 
     private lazy val assistedConfigurationOfNativeImageBuildsLink =
       "https://www.graalvm.org/reference-manual/native-image/BuildConfiguration/#assisted-configuration-of-native-image-builds"
@@ -103,22 +137,33 @@ object NativeImagePlugin extends AutoPlugin {
   override lazy val projectSettings: Seq[Def.Setting[_]] = List(
     libraryDependencies += "org.scalameta" % "svm-subs" % "101.0.0",
     target.in(NativeImage) := target.in(Compile).value / "native-image",
-    target.in(NativeImageInternal) :=
-      target.in(Compile).value / "native-image-internal",
+    target.in(NativeImageTest) := target.in(Test).value / "native-image-test",
+    target.in(NativeImageInternal) := target.in(Compile).value / "native-image-internal",
+    target.in(NativeImageTestInternal) := target.in(Test).value / "native-image-test-internal",
     nativeImageReady := {
       val s = streams.value
 
       { () =>
         this.alertUser(s, "Native image ready!")
       }
     },
-    mainClass.in(NativeImage) := mainClass.in(Compile).value,
+    nativeImageTestReady := {
+      val s = streams.value
+
+      { () =>
+        this.alertUser(s, "Native image of tests is ready!")
+      }
+    },
     nativeImageJvm := "graalvm-java11",
     nativeImageJvmIndex := "cs",
     nativeImageVersion := "20.2.0",
     name.in(NativeImage) := name.value,
+    name.in(NativeImageTest) := name.in(Test).value,
     mainClass.in(NativeImage) := mainClass.in(Compile).value,
+    mainClass.in(NativeImageTest) := mainClass.in(Test).value,
     nativeImageOptions := List(),
+    nativeImageTestOptions := nativeImageOptions.value,
+    nativeImageTestRunOptions := List(),
     nativeImageCoursier := {
       val dir = target.in(NativeImageInternal).value
       val out = copyResource("coursier", dir)
@@ -214,8 +259,11 @@ object NativeImagePlugin extends AutoPlugin {
           }
         }
         .value,
+    nativeImageTestCommand := nativeImageCommand.value,
     nativeImageAgentOutputDir := target.value / "native-image-configs",
+    nativeImageTestAgentOutputDir := nativeImageAgentOutputDir.value,
     nativeImageAgentMerge := false,
+    nativeImageTestAgentMerge := nativeImageAgentMerge.value,
     nativeImageRunAgent := {
       val _ = nativeImageCommand.value
       val graalHome = nativeImageGraalHome.value.toFile
@@ -245,8 +293,54 @@ object NativeImagePlugin extends AutoPlugin {
         .extract(newState)
         .runInputTask(run in (tpr, Compile), input, newState)
     },
+    nativeImageTestRunAgent := {
+      val _ = nativeImageTestCommand.value
+      val graalHome = nativeImageGraalHome.value.toFile
+
+      val agentConfig =
+        if (nativeImageTestAgentMerge.value)
+          "config-merge-dir"
+        else
+          "config-output-dir"
+      val agentOption =
+        s"-agentlib:native-image-agent=$agentConfig=${nativeImageTestAgentOutputDir.value}"
+
+      val options = (javaOptions in (Test, run)).value ++ Seq(agentOption)
+
+      val __ = compile.in(Test).value
+      val main = mainClass.in(NativeImageTest).value
+      val cp = fullClasspath.in(Test).value.map(_.data)
+      val manifest = target.in(NativeImageTestInternal).value / "manifest.jar"
+      manifest.getParentFile().mkdirs()
+      createManifestJar(manifest, cp)
+      val nativeClasspath = manifest.absolutePath
+
+      val command = mutable.ListBuffer.empty[String]
+      command += (graalHome / "bin" / "java").absolutePath
+      command ++= options
+      command += "-cp"
+      command += nativeClasspath
+      command +=
+        main.getOrElse(
+          throw new MessageOnlyException(
+            "no mainClass is specified for tests. " +
+              "To fix this problem, update build.sbt to include the settings " +
+              "`mainClass.in(Test) := Some(\"com.MainTestClass\")`"
+          )
+        )
+      command ++= nativeImageTestRunOptions.value
+
+      val projectRoot = baseDirectory.value
+      streams.value.log.info(command.mkString(" "))
+      val exitCode = Process(command, cwd = Some(projectRoot)).!
+      if (exitCode != 0) {
+        throw new Exception(s"Native image build failed:\n ${command}")
+      }
+    },
     nativeImageOutput :=
       target.in(NativeImage).value / name.in(NativeImage).value,
+    nativeImageTestOutput :=
+      target.in(NativeImageTest).value / name.in(NativeImageTest).value,
     nativeImageCopy := {
       val binary = nativeImage.value
       val out = fileParser(baseDirectory.in(ThisBuild).value).parsed
@@ -270,6 +364,18 @@ object NativeImagePlugin extends AutoPlugin {
         throw new MessageOnlyException(s"non-zero exit: $exit")
       }
     },
+    nativeImageTestRun := {
+      val binary = nativeImageTestOutput.value
+      if (!binary.isFile()) {
+        throw new MessageOnlyException(
+          s"no such file: $binary.\nTo fix this problem, run 'nativeImageTest' first."
+        )
+      }
+      val exit = Process(binary.absolutePath :: nativeImageTestRunOptions.value.toList).!
+      if (exit != 0) {
+        throw new MessageOnlyException(s"non-zero exit: $exit")
+      }
+    },
     nativeImage := {
       val _ = compile.in(Compile).value
       val main = mainClass.in(NativeImage).value
@@ -316,6 +422,53 @@ object NativeImagePlugin extends AutoPlugin {
       nativeImageReady.value.apply()
       streams.value.log.info(binaryName.absolutePath)
       binaryName
+    },
+    nativeImageTest := {
+      val _ = compile.in(Test).value
+      val main = mainClass.in(NativeImageTest).value
+      val binaryName = nativeImageTestOutput.value
+      val cp = fullClasspath.in(Test).value.map(_.data)
+      // NOTE(olafur): we pass in a manifest jar instead of the full classpath
+      // for two reasons:
+      // * large classpaths quickly hit on the "argument list too large"
+      //   error, especially on Windows.
+      // * we print the full command to the console and the manifest jar makes
+      //   it more readable and easier to copy-paste.
+      val manifest = target.in(NativeImageTestInternal).value / "manifest.jar"
+      manifest.getParentFile().mkdirs()
+      createManifestJar(manifest, cp)
+      val nativeClasspath = manifest.absolutePath
+
+      // Assemble native-image argument list.
+      val command = mutable.ListBuffer.empty[String]
+      command ++= nativeImageTestCommand.value
+      command ++= nativeImageTestOptions.value
+      command += "-cp"
+      command += nativeClasspath
+      command +=
+        main.getOrElse(
+          throw new MessageOnlyException(
+            "no mainClass is specified for tests. " +
+              "To fix this problem, update build.sbt to include the settings " +
+              "`mainClass.in(Test) := Some(\"com.MainTestClass\")`"
+          )
+        )
+      command += binaryName.absolutePath
+
+      // Start native-image linker.
+      streams.value.log.info(command.mkString(" "))
+      val cwd = target.in(NativeImageTest).value
+      cwd.mkdirs()
+      val exit = Process(command, cwd = Some(cwd)).!
+      if (exit != 0) {
+        throw new MessageOnlyException(
+          s"native-image command failed with exit code '$exit'"
+        )
+      }
+
+      nativeImageTestReady.value.apply()
+      streams.value.log.info(binaryName.absolutePath)
+      binaryName
     }
   )
 

plugin/src/sbt-test/sbt-native-image/agent-test/build.sbt

@@ -0,0 +1,21 @@
+lazy val example = project
+  .settings(
+    scalaVersion := "2.12.12",
+    mainClass.in(Compile) := Some("example.Hello6"),
+    nativeImageTestOptions ++= Seq(
+      s"-H:ReflectionConfigurationFiles=${target.value / "native-image-configs" / "reflect-config.json"}",
+      "--initialize-at-build-time=scala.collection.immutable.VM",
+    ),
+    mainClass.in(Test) := Some("org.scalatest.tools.Runner"),
+    nativeImageTestRunOptions ++= Seq("-o", "-R", classDirectory.in(Test).value.absolutePath),
+    nativeImageCommand := List(
+      sys.env.getOrElse(
+        "NATIVE_IMAGE_COMMAND",
+        "missing environment variable 'NATIVE_IMAGE_COMMAND'. " +
+          "To fix this problem, manually install GraalVM native-image and update the environment " +
+          "variable to point to the absolute path of this binary."
+      )
+    ),
+    libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.14" % "test"
+  )
+  .enablePlugins(NativeImagePlugin)

plugin/src/sbt-test/sbt-native-image/agent-test/example/src/main/scala/example/Hello6.scala

@@ -0,0 +1,22 @@
+package example
+
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.nio.charset.StandardCharsets
+
+object Hello6 {
+  def main(args: Array[String]): Unit = {
+    val cl = this.getClass.getClassLoader
+    val c = cl.loadClass("example.Hello6")
+    val h3 = c.getConstructor().newInstance()
+    val text = h3.toString
+    Files.write(
+      Paths.get("hello6.obtained"),
+      text.getBytes(StandardCharsets.UTF_8)
+    )
+  }
+}
+
+class Hello6 {
+  override def toString: String = "Hello 6"
+}

plugin/src/sbt-test/sbt-native-image/agent-test/example/src/test/scala/example/Hello6Spec.scala

@@ -0,0 +1,23 @@
+package example
+
+import org.scalatest.flatspec.AnyFlatSpec
+
+import java.io.File
+import java.nio.charset.StandardCharsets
+import java.nio.file.{Files, Paths, StandardOpenOption}
+
+class Hello6Spec extends AnyFlatSpec {
+
+  behavior of "Hello6"
+
+  it should "append Hello6 output" in {
+    Hello6.main(Array.empty)
+    assert(new File("Hello6.obtained").exists())
+
+    Files.write(
+      Paths.get("Hello6.obtained"),
+      "-tested".getBytes(StandardCharsets.UTF_8),
+      StandardOpenOption.APPEND
+    )
+  }
+}

plugin/src/sbt-test/sbt-native-image/agent-test/hello6.expected

@@ -0,0 +1 @@
+Hello 6-tested

plugin/src/sbt-test/sbt-native-image/agent-test/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("org.scalameta" % "sbt-native-image" % sys.props("plugin.version"))

plugin/src/sbt-test/sbt-native-image/agent-test/test

@@ -0,0 +1,6 @@
+> example/nativeImageTestRunAgent
+$ delete hello6.obtained
+> example/nativeImageTest
+$ absent hello6.obtained
+> example/nativeImageTestRun
+$ must-mirror hello6.expected hello6.obtained

plugin/src/sbt-test/sbt-native-image/basic-test/build.sbt

@@ -0,0 +1,20 @@
+lazy val example = project
+  .settings(
+    scalaVersion := "2.12.12",
+    mainClass.in(Compile) := Some("example.Hello4"),
+    nativeImageCommand := List(
+      sys.env.getOrElse(
+        "NATIVE_IMAGE_COMMAND",
+        "missing environment variable 'NATIVE_IMAGE_COMMAND'. " +
+          "To fix this problem, manually install GraalVM native-image and update the environment " +
+          "variable to point to the absolute path of this binary."
+      )
+    ),
+    nativeImageTestOptions ++= Seq(
+      "--initialize-at-build-time=scala.collection.immutable.VM"
+    ),
+    mainClass.in(Test) := Some("org.scalatest.tools.Runner"),
+    nativeImageTestRunOptions ++= Seq("-o", "-R", classDirectory.in(Test).value.absolutePath),
+    libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.14" % "test"
+  )
+  .enablePlugins(NativeImagePlugin)

plugin/src/sbt-test/sbt-native-image/basic-test/example/src/main/scala/example/Hello4.scala

@@ -0,0 +1,15 @@
+package example
+
+import java.nio.file.Files
+import java.nio.file.Paths
+import java.nio.charset.StandardCharsets
+
+object Hello4 {
+  def main(args: Array[String]): Unit = {
+    val text = List(1, 2, 3, 4).toString()
+    Files.write(
+      Paths.get("hello4.obtained"),
+      text.getBytes(StandardCharsets.UTF_8)
+    )
+  }
+}