Generalize Zinc Worker (#514)

* Generalize Zinc worker - Compiler bridges can now be either pre-compiled or on-demand-compiled - Scala library/compiler jar discovery is now configurable - Zinc compiler cache is now configurable, rather than being hardcoded at n=1 * . * update constructor args * remove duplicate util/AggWrapper.scala file * fix * fix * fix * cleanup
com-lihaoyi · Dec 19, 2018 · de175e6 · de175e6
1 parent ea36ea3
commit de175e6
Show file tree

Hide file tree

Showing 33 changed files with 248 additions and 291 deletions.
diff --git a/ci/shared.sc b/ci/shared.sc
@@ -74,7 +74,7 @@ def generateEval(dir: Path) = {
     s"""package mill.main
        |import mill.eval.Evaluator
        |import mill.define.Task
-       |import mill.util.Strict.Agg
+       |import mill.api.Strict.Agg
        |class EvalGenerated(evaluator: Evaluator) {
        |  type TT[+X] = Task[X]
        |  ${(1 to 22).map(generate).mkString("\n")}

diff --git a/contrib/scalapblib/src/ScalaPBModule.scala b/contrib/scalapblib/src/ScalaPBModule.scala
@@ -7,7 +7,7 @@ import mill.define.Sources
 import mill.api.PathRef
 import mill.scalalib.Lib.resolveDependencies
 import mill.scalalib._
-import mill.util.Loose
+import mill.api.Loose
 
 trait ScalaPBModule extends ScalaModule {
 

diff --git a/contrib/twirllib/src/TwirlModule.scala b/contrib/twirllib/src/TwirlModule.scala
@@ -6,7 +6,7 @@ import mill.define.Sources
 import mill.api.PathRef
 import mill.scalalib.Lib.resolveDependencies
 import mill.scalalib._
-import mill.util.Loose
+import mill.api.Loose
 
 import scala.io.Codec
 import scala.util.Properties

diff --git a/main/api/src/mill/api/KeyedLockedCache.scala b/main/api/src/mill/api/KeyedLockedCache.scala
@@ -0,0 +1,44 @@
+package mill.api
+
+/**
+  * A combination lock & cache; users provide a key, value-factory, and a
+  * body function to be called with the value. [[KeyedLockedCache]] ensures that
+  * the body function is called with the computed/cached value sequentially.
+  */
+trait KeyedLockedCache[T]{
+  def withCachedValue[V](key: Long)(f: => T)(f2: T => V): V
+}
+
+object KeyedLockedCache{
+  class RandomBoundedCache[T](hotParallelism: Int, coldCacheSize: Int) extends KeyedLockedCache[T]{
+    private[this] val random = new scala.util.Random(313373)
+    val available = new java.util.concurrent.Semaphore(hotParallelism)
+
+    // Awful asymptotic complexity, but our caches are tiny n < 10 so it doesn't matter
+    var cache = Array.fill[Option[(Long, T)]](coldCacheSize)(None)
+
+    def withCachedValue[V](key: Long)(f: => T)(f2: T => V): V = {
+      available.acquire()
+      val pickedValue = synchronized{
+        cache.indexWhere(_.exists(_._1 == key)) match {
+          case -1 => f
+          case i =>
+            val (k, v) = cache(i).get
+            cache(i) = None
+            v
+        }
+      }
+      val result = f2(pickedValue)
+      synchronized{
+        cache.indexWhere(_.isEmpty) match{
+          // Random eviction #YOLO
+          case -1 => cache(random.nextInt(cache.length)) = Some((key, pickedValue))
+          case i => cache(i) = Some((key, pickedValue))
+        }
+      }
+
+      available.release()
+      result
+    }
+  }
+}
diff --git a/main/core/src/define/Graph.scala b/main/core/src/define/Graph.scala
@@ -2,7 +2,7 @@ package mill.define
 
 import mill.eval.Tarjans
 import mill.util.MultiBiMap
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 
 object Graph {
 

diff --git a/main/core/src/eval/Evaluator.scala b/main/core/src/eval/Evaluator.scala
@@ -10,7 +10,7 @@ import mill.define.{Ctx => _, _}
 import mill.api.Result.OuterStack
 import mill.util
 import mill.util._
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 
 import scala.collection.mutable
 import scala.util.control.NonFatal

diff --git a/main/core/src/util/AggWrapper.scala b/main/core/src/util/AggWrapper.scala
diff --git a/main/core/src/util/MultiBiMap.scala b/main/core/src/util/MultiBiMap.scala
@@ -1,7 +1,7 @@
 package mill.util
 
 import scala.collection.mutable
-import Strict.Agg
+import mill.api.Strict.Agg
 
 /**
   * A map from keys to collections of values: you can assign multiple values

diff --git a/main/src/main/ReplApplyHandler.scala b/main/src/main/ReplApplyHandler.scala
@@ -6,7 +6,7 @@ import mill.define.Segment.Label
 import mill.define._
 import mill.eval.{Evaluator, Result}
 
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 
 import scala.collection.mutable
 object ReplApplyHandler{

diff --git a/main/src/main/RunScript.scala b/main/src/main/RunScript.scala
@@ -11,7 +11,7 @@ import mill.define._
 import mill.eval.{Evaluator, PathRef, Result}
 import mill.util.{EitherOps, ParseArgs, Watched}
 import mill.api.Logger
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 
 import scala.collection.mutable
 import scala.reflect.ClassTag

diff --git a/main/src/modules/Jvm.scala b/main/src/modules/Jvm.scala
@@ -15,7 +15,7 @@ import mill.main.client.InputPumper
 import mill.eval.{PathRef, Result}
 import mill.util.Ctx
 import mill.api.IO
-import mill.util.Loose.Agg
+import mill.api.Loose.Agg
 
 import scala.collection.mutable
 import scala.collection.JavaConverters._

diff --git a/main/src/modules/Util.scala b/main/src/modules/Util.scala
@@ -3,7 +3,8 @@ package mill.modules
 
 import coursier.Repository
 import mill.api.{PathRef, IO}
-import mill.util.{Ctx, Loose}
+import mill.util.Ctx
+import mill.api.Loose
 
 
 object Util {
@@ -55,7 +56,7 @@ object Util {
     val localPath = sys.props(key)
     if (localPath != null) {
       mill.api.Result.Success(
-        Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true)))
+        mill.api.Loose.Agg.from(localPath.split(',').map(p => PathRef(os.Path(p), quick = true)))
       )
     } else {
       mill.modules.Jvm.resolveDependencies(

diff --git a/main/src/package.scala b/main/src/package.scala
@@ -7,6 +7,6 @@ package object mill extends JsonFormatters{
   type PathRef = mill.api.PathRef
   type Module = define.Module
   type Cross[T] = define.Cross[T]
-  type Agg[T] = util.Loose.Agg[T]
-  val Agg = util.Loose.Agg
+  type Agg[T] = mill.api.Loose.Agg[T]
+  val Agg = mill.api.Loose.Agg
 }
diff --git a/main/test/resources/examples/javac/build.sc b/main/test/resources/examples/javac/build.sc
@@ -2,7 +2,7 @@ import mill.T
 import mill.eval.JavaCompileJarTests.compileAll
 import mill.api.PathRef
 import mill.modules.Jvm
-import mill.util.Loose
+import mill.api.Loose
 
 def sourceRootPath = millSourcePath / 'src
 def resourceRootPath = millSourcePath / 'resources

diff --git a/main/test/src/define/CacherTests.scala b/main/test/src/define/CacherTests.scala
@@ -1,7 +1,7 @@
 package mill.define
 
 import mill.util.{DummyLogger, TestEvaluator, TestUtil}
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 import mill.T
 import mill.api.Result.Success
 import utest._

diff --git a/main/test/src/define/GraphTests.scala b/main/test/src/define/GraphTests.scala
@@ -4,7 +4,7 @@ package mill.define
 import mill.eval.Evaluator
 import mill.util.{TestGraphs, TestUtil}
 import utest._
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 object GraphTests extends TestSuite{
 
   val tests = Tests{

diff --git a/main/test/src/eval/EvaluationTests.scala b/main/test/src/eval/EvaluationTests.scala
@@ -5,7 +5,7 @@ import mill.util.TestUtil.{Test, test}
 import mill.define.{Discover, Graph, Target, Task}
 import mill.{Module, T}
 import mill.util.{DummyLogger, TestEvaluator, TestGraphs, TestUtil}
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 import utest._
 import utest.framework.TestPath
 

diff --git a/main/test/src/eval/JavaCompileJarTests.scala b/main/test/src/eval/JavaCompileJarTests.scala
@@ -4,12 +4,13 @@ import mill.define.{Discover, Input, Target, Task}
 import mill.modules.Jvm
 import mill.api.Ctx.Dest
 import mill.{Module, T}
-import mill.util.{DummyLogger, Loose, TestEvaluator, TestUtil}
-import mill.util.Strict.Agg
+import mill.util.{DummyLogger, TestEvaluator, TestUtil}
+import mill.api.Strict.Agg
+import mill.api.Loose
 import utest._
 import mill._
 object JavaCompileJarTests extends TestSuite{
-  def compileAll(sources: mill.util.Loose.Agg[PathRef])(implicit ctx: Dest) = {
+  def compileAll(sources: mill.api.Loose.Agg[PathRef])(implicit ctx: Dest) = {
     os.makeDir.all(ctx.dest)
 
     os.proc("javac", sources.map(_.path.toString()).toSeq, "-d", ctx.dest).call(ctx.dest)

diff --git a/main/test/src/util/TestEvaluator.scala b/main/test/src/util/TestEvaluator.scala
@@ -3,7 +3,7 @@ package mill.util
 import mill.define.{Input, Target, Task}
 import mill.api.Result.OuterStack
 import mill.eval.{Evaluator, Result}
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 import utest.assert
 import utest.framework.TestPath
 

diff --git a/main/test/src/util/TestUtil.scala b/main/test/src/util/TestUtil.scala
@@ -5,7 +5,7 @@ import mill.define._
 import mill.api.Result
 import mill.api.Result.OuterStack
 import utest.assert
-import mill.util.Strict.Agg
+import mill.api.Strict.Agg
 import utest.framework.TestPath
 
 import scala.collection.mutable

diff --git a/scalajslib/src/ScalaJSModule.scala b/scalajslib/src/ScalaJSModule.scala
@@ -7,7 +7,8 @@ import mill.eval.{PathRef, Result}
 import mill.api.Result.Success
 import mill.scalalib.Lib.resolveDependencies
 import mill.scalalib.{DepSyntax, Lib, TestModule, TestRunner}
-import mill.util.{Ctx, Loose}
+import mill.util.Ctx
+import mill.api.Loose
 import mill.scalajslib.api._
 trait ScalaJSModule extends scalalib.ScalaModule { outer =>