diff --git a/benchmark/src/main/scala/firrtl/benchmark/hot/ResolveKindsBenchmark.scala b/benchmark/src/main/scala/firrtl/benchmark/hot/ResolveKindsBenchmark.scala new file mode 100644 index 0000000000..61e9c1a64b --- /dev/null +++ b/benchmark/src/main/scala/firrtl/benchmark/hot/ResolveKindsBenchmark.scala @@ -0,0 +1,23 @@ + +package firrtl +package benchmark +package hot + +import passes.ResolveKinds +import stage.TransformManager + +import firrtl.benchmark.util._ + +object ResolveKindsBenchmark extends App { + val inputFile = args(0) + val warmup = args(1).toInt + val runs = args(2).toInt + + val input = filenameToCircuit(inputFile) + val state = CircuitState(input, ChirrtlForm) + val prereqs = ResolveKinds.prerequisites + val manager = new TransformManager(prereqs) + val preState = manager.execute(state) + + hot.util.benchmark(warmup, runs)(ResolveKinds.run(preState.circuit)) +} diff --git a/benchmark/src/main/scala/firrtl/benchmark/hot/util/package.scala b/benchmark/src/main/scala/firrtl/benchmark/hot/util/package.scala new file mode 100644 index 0000000000..c05b83597c --- /dev/null +++ b/benchmark/src/main/scala/firrtl/benchmark/hot/util/package.scala @@ -0,0 +1,28 @@ + +package firrtl.benchmark.hot + +import firrtl.Utils.time +import firrtl.benchmark.util._ + +package object util { + def benchmark(nWarmup: Int, nRun: Int)(f: => Unit): Unit = { + // Warmup + for (i <- 0 until nWarmup) { + val (t, res) = time(f) + println(f"Warmup run $i took $t%.1f ms") + } + + // Benchmark + val times: Array[Double] = Array.fill(nRun)(0.0) + for (i <- 0 until nRun) { + val (t, res) = time(f) + times(i) = t + println(f"Benchmark run $i took $t%.1f ms") + } + + println(f"Mean: ${mean(times)}%.1f ms") + println(f"Median: ${median(times)}%.1f ms") + println(f"Stddev: ${stdDev(times)}%.1f ms") + } + +} diff --git a/benchmark/src/main/scala/firrtl/benchmark/util/package.scala b/benchmark/src/main/scala/firrtl/benchmark/util/package.scala new file mode 100644 index 0000000000..2923d8b5fe --- /dev/null +++ b/benchmark/src/main/scala/firrtl/benchmark/util/package.scala @@ -0,0 +1,34 @@ + +package firrtl +package benchmark + +import firrtl.ir.Circuit +import scala.util.control.NonFatal + +package object util { + def filenameToCircuit(filename: String): Circuit = try { + proto.FromProto.fromFile(filename) + } catch { + case NonFatal(_) => Parser.parseFile(filename, Parser.IgnoreInfo) + } + + def mean(xs: Iterable[Double]): Double = xs.sum / xs.size + + def median(xs: Iterable[Double]): Double = { + val size = xs.size + val sorted = xs.toSeq.sorted + if (size % 2 == 1) sorted(size / 2) + else { + val a = sorted(size / 2) + val b = sorted((size / 2) - 1) + (a + b) / 2 + } + } + + def variance(xs: Iterable[Double]): Double = { + val avg = mean(xs) + xs.map(a => math.pow(a - avg, 2)).sum / xs.size + } + + def stdDev(xs: Iterable[Double]): Double = math.sqrt(variance(xs)) +} diff --git a/build.sbt b/build.sbt index c4705019db..3fc859a5ab 100644 --- a/build.sbt +++ b/build.sbt @@ -185,3 +185,11 @@ lazy val firrtl = (project in file(".")) .settings(testAssemblySettings) .settings(publishSettings) .settings(docSettings) + +lazy val benchmark = (project in file("benchmark")) + .dependsOn(firrtl) + .settings( + assemblyJarName in assembly := "firrtl-benchmark.jar", + test in assembly := {}, + assemblyOutputPath in assembly := file("./utils/bin/firrtl-benchmark.jar") + )