Add benchmark for ResolveKinds with hot JIT

benchmark/src/main/scala/firrtl/benchmark/hot/ResolveKindsBenchmark.scala

@@ -0,0 +1,23 @@
+
+package firrtl
+package benchmark
+package hot
+
+import passes.ResolveKinds
+import stage.TransformManager
+
+import firrtl.benchmark.util._
+
+object ResolveKindsBenchmark extends App {
+  val inputFile = args(0)
+  val warmup = args(1).toInt
+  val runs = args(2).toInt
+
+  val input = filenameToCircuit(inputFile)
+  val state = CircuitState(input, ChirrtlForm)
+  val prereqs = ResolveKinds.prerequisites
+  val manager = new TransformManager(prereqs)
+  val preState = manager.execute(state)
+
+  hot.util.benchmark(warmup, runs)(ResolveKinds.run(preState.circuit))
+}

benchmark/src/main/scala/firrtl/benchmark/hot/util/package.scala

@@ -0,0 +1,28 @@
+
+package firrtl.benchmark.hot
+
+import firrtl.Utils.time
+import firrtl.benchmark.util._
+
+package object util {
+  def benchmark(nWarmup: Int, nRun: Int)(f: => Unit): Unit = {
+    // Warmup
+    for (i <- 0 until nWarmup) {
+      val (t, res) = time(f)
+      println(f"Warmup run $i took $t%.1f ms")
+    }
+
+    // Benchmark
+    val times: Array[Double] = Array.fill(nRun)(0.0)
+    for (i <- 0 until nRun) {
+      val (t, res) = time(f)
+      times(i) = t
+      println(f"Benchmark run $i took $t%.1f ms")
+    }
+
+    println(f"Mean:   ${mean(times)}%.1f ms")
+    println(f"Median: ${median(times)}%.1f ms")
+    println(f"Stddev: ${stdDev(times)}%.1f ms")
+  }
+
+}

benchmark/src/main/scala/firrtl/benchmark/util/package.scala

@@ -0,0 +1,34 @@
+
+package firrtl
+package benchmark
+
+import firrtl.ir.Circuit
+import scala.util.control.NonFatal
+
+package object util {
+  def filenameToCircuit(filename: String): Circuit = try {
+    proto.FromProto.fromFile(filename)
+  } catch {
+    case NonFatal(_) => Parser.parseFile(filename, Parser.IgnoreInfo)
+  }
+
+  def mean(xs: Iterable[Double]): Double = xs.sum / xs.size
+
+  def median(xs: Iterable[Double]): Double = {
+    val size = xs.size
+    val sorted = xs.toSeq.sorted
+    if (size % 2 == 1) sorted(size / 2)
+    else {
+      val a = sorted(size / 2)
+      val b = sorted((size / 2) - 1)
+      (a + b) / 2
+    }
+  }
+
+  def variance(xs: Iterable[Double]): Double = {
+    val avg = mean(xs)
+    xs.map(a => math.pow(a - avg, 2)).sum / xs.size
+  }
+
+  def stdDev(xs: Iterable[Double]): Double = math.sqrt(variance(xs))
+}

build.sbt

@@ -184,3 +184,11 @@ lazy val firrtl = (project in file("."))
   .settings(testAssemblySettings)
   .settings(publishSettings)
   .settings(docSettings)
+
+lazy val benchmark = (project in file("benchmark"))
+  .dependsOn(firrtl)
+  .settings(
+    assemblyJarName in assembly := "firrtl-benchmark.jar",
+    test in assembly := {},
+    assemblyOutputPath in assembly := file("./utils/bin/firrtl-benchmark.jar")
+  )