Skip to content

Commit

Permalink
Implement remote test caching
Browse files Browse the repository at this point in the history
  • Loading branch information
eed3si9n committed Sep 8, 2024
1 parent 2aba06b commit 7929eba
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 28 deletions.
2 changes: 1 addition & 1 deletion main/src/main/scala/sbt/Defaults.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1413,8 +1413,8 @@ object Defaults extends BuildCommon {
Keys.logLevel.?.value.getOrElse(stateLogLevel),
) +:
TestStatusReporter(
IncrementalTest.succeededFile((test / streams).value.cacheDirectory),
definedTestDigests.value,
Def.cacheConfiguration.value,
) +:
(TaskZero / testListeners).value
},
Expand Down
14 changes: 0 additions & 14 deletions main/src/main/scala/sbt/RemoteCache.scala
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,6 @@ object RemoteCache {
) ++ inConfig(Compile)(
configCacheSettings(compileArtifact(Compile, cachedCompileClassifier))
)
++ inConfig(Test)(configCacheSettings(testArtifact(Test, cachedTestClassifier))))

def getResourceFilePaths() = Def.task {
val syncDir = crossTarget.value / (prefix(configuration.value.name) + "sync")
Expand Down Expand Up @@ -383,19 +382,6 @@ object RemoteCache {
)
}

def testArtifact(
configuration: Configuration,
classifier: String
): Def.Initialize[Task[TestRemoteCacheArtifact]] = Def.task {
TestRemoteCacheArtifact(
Artifact(moduleName.value, classifier),
configuration / packageCache,
(configuration / classDirectory).value,
(configuration / compileAnalysisFile).value,
IncrementalTest.succeededFile((configuration / test / streams).value.cacheDirectory)
)
}

private def toVersion(v: String): String = s"0.0.0-$v"

private lazy val doption = new DownloadOptions
Expand Down
51 changes: 38 additions & 13 deletions main/src/main/scala/sbt/internal/IncrementalTest.scala
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import sbt.io.syntax.*
import sbt.io.{ GlobFilter, IO, NameFilter }
import sbt.protocol.testing.TestResult
import sbt.SlashSyntax0.*
import sbt.util.Digest
import sbt.util.{ ActionCache, BuildWideCacheConfiguration, CacheLevelTag, Digest }
import sbt.util.CacheImplicits.given
import scala.collection.concurrent
import scala.collection.mutable
Expand All @@ -33,10 +33,13 @@ object IncrementalTest:
val cp = (Keys.test / fullClasspath).value
val s = (Keys.test / streams).value
val digests = (Keys.definedTestDigests).value
val succeeded = TestStatus.read(succeededFile(s.cacheDirectory))
def hasSucceeded(className: String): Boolean = succeeded.get(className) match
val config = Def.cacheConfiguration.value
def hasCachedSuccess(ts: Digest): Boolean =
val input = cacheInput(ts)
ActionCache.exists(input._1, input._2, input._3, config)
def hasSucceeded(className: String): Boolean = digests.get(className) match
case None => false
case Some(ts) => Some(ts) == digests.get(className)
case Some(ts) => hasCachedSuccess(ts)
args =>
for filter <- selectedFilter(args)
yield (test: String) => filter(test) && !hasSucceeded(test)
Expand All @@ -47,19 +50,25 @@ object IncrementalTest:
val cp = (Keys.test / fullClasspath).value
val testNames = Keys.definedTests.value.map(_.name).toVector.distinct
val converter = fileConverter.value
val inputs = Keys.compileInputs.value
val extra = Digest(converter.toVirtualFile(inputs.options.classesDirectory))
val sv = Keys.scalaVersion.value
val inputs = (Keys.compile / Keys.compileInputs).value
// by default this captures JVM version
val extraInc = Keys.extraIncOptions.value
// throw in any information useful for runtime invalidation
val salt = s"""$sv
${converter.toVirtualFile(inputs.options.classesDirectory)}
${extraInc.mkString(",")}
"""
val extra = Vector(Digest.sha256Hash(salt.getBytes("UTF-8")))
val stamper = ClassStamper(cp, converter)
// TODO: Potentially do something about JUnit 5 and others which might not use class name
Map((testNames.flatMap: name =>
stamper.transitiveStamp(name, Vector(extra)) match
stamper.transitiveStamp(name, extra) match
case Some(ts) => Seq(name -> ts)
case None => Nil
): _*)
}

def succeededFile(dir: File): File = dir / "succeeded_tests.txt"

def selectedFilter(args: Seq[String]): Seq[String => Boolean] =
def matches(nfs: Seq[NameFilter], s: String) = nfs.exists(_.accept(s))
val (excludeArgs, includeArgs) = args.partition(_.startsWith("-"))
Expand All @@ -70,13 +79,17 @@ object IncrementalTest:
case (Nil, _) => Seq((s: String) => !matches(excludeFilters, s))
case _ =>
includeFilters.map(f => (s: String) => (f.accept(s) && !matches(excludeFilters, s)))

private[sbt] def cacheInput(value: Digest): (Unit, Digest, Digest) =
((), value, Digest.zero)
end IncrementalTest

// Assumes exclusive ownership of the file.
private[sbt] class TestStatusReporter(
f: File,
digests: Map[String, Digest],
cacheConfiguration: BuildWideCacheConfiguration,
) extends TestsListener:
// int value to represent success
private final val successfulTest = 0
private lazy val succeeded: concurrent.Map[String, Digest] =
TestStatus.read(f)

Expand All @@ -94,8 +107,20 @@ private[sbt] class TestStatusReporter(
def endGroup(name: String, result: TestResult): Unit =
if result == TestResult.Passed then
digests.get(name) match
case Some(ts) => succeeded(name) = ts
case None => succeeded(name) = Digest.zero
case Some(ts) =>
succeeded(name) = ts
// treat each test suite as a successful action that returns 0
val input = IncrementalTest.cacheInput(ts)
ActionCache.cache(
key = input._1,
codeContentHash = input._2,
extraHash = input._3,
tags = CacheLevelTag.all.toList,
config = cacheConfiguration,
): (_) =>
ActionCache.actionResult(successfulTest)
case None =>
succeeded(name) = Digest.zero
else ()
def doComplete(finalResult: TestResult): Unit =
TestStatus.write(succeeded, "Successful Tests", f)
Expand Down

0 comments on commit 7929eba

Please sign in to comment.