From 2b159274b909a70e707f0f0acbf556eec7f0dd63 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 12 Nov 2023 02:41:12 -0500 Subject: [PATCH] Virtualize packageBin, part 3 --- main-actions/src/main/scala/sbt/Pkg.scala | 25 +++++--- main/src/main/scala/sbt/Defaults.scala | 59 ++++++++++++------- main/src/main/scala/sbt/Keys.scala | 3 +- .../scala/sbt/util/ActionCacheStore.scala | 5 +- .../scala/sbt/util/HashedVirtualFileRef.scala | 4 ++ .../scala/sbt/util/BasicCacheImplicits.scala | 1 - 6 files changed, 65 insertions(+), 32 deletions(-) diff --git a/main-actions/src/main/scala/sbt/Pkg.scala b/main-actions/src/main/scala/sbt/Pkg.scala index 89f5ffd682..a639a0d3a2 100644 --- a/main-actions/src/main/scala/sbt/Pkg.scala +++ b/main-actions/src/main/scala/sbt/Pkg.scala @@ -26,13 +26,19 @@ import sjsonnew.{ } import sbt.util.Logger - -import sbt.util.{ CacheStoreFactory, FilesInfo, ModifiedFileInfo, PlainFileInfo } +import sbt.util.{ + CacheStoreFactory, + FilesInfo, + HashedVirtualFileRef, + ModifiedFileInfo, + PlainFileInfo +} import sbt.util.FileInfo.{ exists, lastModified } import sbt.util.CacheImplicits._ import sbt.util.Tracked.{ inputChanged, outputChanged } import scala.sys.process.Process import xsbti.FileConverter +import xsbti.VirtualFile import xsbti.VirtualFileRef /** @@ -106,7 +112,7 @@ object Pkg: * @param options additional package information, e.g. jar manifest, main class or manifest attributes */ final class Configuration( - val sources: Seq[(VirtualFileRef, String)], + val sources: Seq[(HashedVirtualFileRef, String)], val jar: VirtualFileRef, val options: Seq[PackageOption] ) @@ -114,14 +120,15 @@ object Pkg: object Configuration: given IsoLList.Aux[ Configuration, - Vector[(VirtualFileRef, String)] :*: VirtualFileRef :*: Seq[PackageOption] :*: LNil + Vector[(HashedVirtualFileRef, String)] :*: VirtualFileRef :*: Seq[PackageOption] :*: LNil ] = import sbt.util.CacheImplicits.given + import sbt.util.PathHashWriters.given LList.iso( (c: Configuration) => ("sources", c.sources.toVector) :*: ("jar", c.jar) :*: ("options", c.options) :*: LNil, - (in: Vector[(VirtualFileRef, String)] :*: VirtualFileRef :*: Seq[PackageOption] :*: LNil) => - Configuration(in.head, in.tail.head, in.tail.tail.head), + (in: Vector[(HashedVirtualFileRef, String)] :*: VirtualFileRef :*: Seq[PackageOption] :*: + LNil) => Configuration(in.head, in.tail.head, in.tail.tail.head), ) given JsonFormat[Configuration] = summon[JsonFormat[Configuration]] end Configuration @@ -131,7 +138,7 @@ object Pkg: * @param cacheStoreFactory used for jar caching. We try to avoid rebuilds as much as possible * @param log feedback for the user */ - def apply(conf: Configuration, converter: FileConverter, log: Logger): Unit = + def apply(conf: Configuration, converter: FileConverter, log: Logger): VirtualFile = apply(conf, converter, log, timeFromConfiguration(conf)) /** @@ -145,14 +152,14 @@ object Pkg: converter: FileConverter, log: Logger, time: Option[Long] - ): Unit = { + ): VirtualFile = val manifest = toManifest(conf, log) val out = converter.toPath(conf.jar).toFile() val sources = conf.sources.map { case (vf, path) => converter.toPath(vf).toFile() -> path } makeJar(sources, out, manifest, log, time) - } + converter.toVirtualFile(out.toPath()) def toManifest(conf: Configuration, log: Logger): Manifest = val manifest = new Manifest diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 3bad9b5530..4232a082fb 100644 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -129,6 +129,7 @@ import xsbti.compile.{ Setup, TransactionalManagerType } +import sbt.util.HashedVirtualFileRef object Defaults extends BuildCommon { final val CacheDirectoryName = "cache" @@ -421,7 +422,7 @@ object Defaults extends BuildCommon { val coursierCache = csrCacheDirectory.value val javaHome = Paths.get(sys.props("java.home")) val out = rootOutputDirectory.value - Map( + ListMap( "OUT" -> out, "BASE" -> base.toPath, "SBT_BOOT" -> boot.toPath, @@ -1752,39 +1753,54 @@ object Defaults extends BuildCommon { packageTaskSettings(packageDoc, packageDocMappings) ++ Seq(Keys.`package` := packageBin.value) - def packageBinMappings: Initialize[Task[Seq[(VirtualFileRef, String)]]] = + def packageBinMappings: Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] = Def.task { val converter = fileConverter.value val xs = products.value xs .flatMap(Path.allSubpaths) - .map { case (p, path) => converter.toVirtualFile(p.toPath()) -> path } + .filter(_._1.isFile()) + .map { case (p, path) => + val vf = converter.toVirtualFile(p.toPath()) + HashedVirtualFileRef.of(vf) -> path + } } - def packageDocMappings: Initialize[Task[Seq[(VirtualFileRef, String)]]] = + def packageDocMappings: Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] = Def.task { val converter = fileConverter.value val d = doc.value - Path.allSubpaths(d).toSeq.map { case (p, path) => - converter.toVirtualFile(p.toPath()) -> path - } + Path + .allSubpaths(d) + .toSeq + .filter(_._1.isFile()) + .map { case (p, path) => + val vf = converter.toVirtualFile(p.toPath()) + HashedVirtualFileRef.of(vf) -> path + } } - def packageSrcMappings: Initialize[Task[Seq[(VirtualFileRef, String)]]] = + def packageSrcMappings: Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] = concatMappings(resourceMappings, sourceMappings) - private type Mappings = Initialize[Task[Seq[(VirtualFileRef, String)]]] + private type Mappings = Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] def concatMappings(as: Mappings, bs: Mappings): Mappings = as.zipWith(bs) { - (a: Task[Seq[(VirtualFileRef, String)]], b: Task[Seq[(VirtualFileRef, String)]]) => + ( + a: Task[Seq[(HashedVirtualFileRef, String)]], + b: Task[Seq[(HashedVirtualFileRef, String)]] + ) => (a, b).mapN { - case (seq1: Seq[(VirtualFileRef, String)], seq2: Seq[(VirtualFileRef, String)]) => + case ( + seq1: Seq[(HashedVirtualFileRef, String)], + seq2: Seq[(HashedVirtualFileRef, String)] + ) => seq1 ++ seq2 } } // drop base directories, since there are no valid mappings for these - def sourceMappings: Initialize[Task[Seq[(VirtualFileRef, String)]]] = + def sourceMappings: Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] = Def.task { val converter = fileConverter.value val sdirs = unmanagedSourceDirectories.value @@ -1797,17 +1813,18 @@ object Defaults extends BuildCommon { case _ => None } .map { case (p, path) => - converter.toVirtualFile(p.toPath()) -> path + val vf = converter.toVirtualFile(p.toPath()) + HashedVirtualFileRef.of(vf) -> path } } - def resourceMappings: Initialize[Task[Seq[(VirtualFileRef, String)]]] = + def resourceMappings: Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] = relativeMappings(unmanagedResources, unmanagedResourceDirectories) def relativeMappings( files: Taskable[Seq[File]], dirs: Taskable[Seq[File]] - ): Initialize[Task[Seq[(VirtualFileRef, String)]]] = + ): Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] = Def.task { val converter = fileConverter.value val rdirs = dirs.toTask.value.toSet @@ -1818,7 +1835,8 @@ object Defaults extends BuildCommon { case _ => None } .map { case (p, path) => - converter.toVirtualFile(p.toPath()) -> path + val vf = converter.toVirtualFile(p.toPath()) + HashedVirtualFileRef.of(vf) -> path } } @@ -1834,7 +1852,7 @@ object Defaults extends BuildCommon { def relativeMappings( // forward to widened variant files: ScopedTaskable[Seq[File]], dirs: ScopedTaskable[Seq[File]] - ): Initialize[Task[Seq[(VirtualFileRef, String)]]] = + ): Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] = relativeMappings(files: Taskable[Seq[File]], dirs) def collectFiles( // forward to widened variant @@ -1933,7 +1951,7 @@ object Defaults extends BuildCommon { def packageTaskSettings( key: TaskKey[VirtualFileRef], - mappingsTask: Initialize[Task[Seq[(VirtualFileRef, String)]]] + mappingsTask: Initialize[Task[Seq[(HashedVirtualFileRef, String)]]] ) = inTask(key)( Seq( @@ -1951,13 +1969,14 @@ object Defaults extends BuildCommon { val config = packageConfiguration.value val s = streams.value val converter = fileConverter.value - Pkg( + val out = Pkg( config, converter, s.log, Pkg.timeFromConfiguration(config) ) - config.jar + Def.declareOutput(out) + out } def packageConfigurationTask: Initialize[Task[Pkg.Configuration]] = diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index b99bb2223a..887be8e2e8 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -43,6 +43,7 @@ import xsbti.compile.analysis.ReadStamps import scala.concurrent.duration.{ Duration, FiniteDuration } import scala.xml.{ NodeSeq, Node => XNode } +import sbt.util.HashedVirtualFileRef // format: off @@ -281,7 +282,7 @@ object Keys { val artifact = settingKey[Artifact]("Describes an artifact.").withRank(BMinusSetting) val artifactClassifier = settingKey[Option[String]]("Sets the classifier used by the default artifact definition.").withRank(BSetting) val artifactName = settingKey[(ScalaVersion, ModuleID, Artifact) => String]("Function that produces the artifact name from its definition.").withRank(CSetting) - val mappings = taskKey[Seq[(VirtualFileRef, String)]]("Defines the mappings from a file to a path, used by packaging, for example.").withRank(BTask) + val mappings = taskKey[Seq[(HashedVirtualFileRef, String)]]("Defines the mappings from a file to a path, used by packaging, for example.").withRank(BTask) val fileMappings = taskKey[Seq[(File, File)]]("Defines the mappings from a file to a file, used for copying files, for example.").withRank(BMinusTask) // Run Keys diff --git a/util-cache-resolver/src/main/scala/sbt/util/ActionCacheStore.scala b/util-cache-resolver/src/main/scala/sbt/util/ActionCacheStore.scala index 8faebfe74d..f0b9b09569 100644 --- a/util-cache-resolver/src/main/scala/sbt/util/ActionCacheStore.scala +++ b/util-cache-resolver/src/main/scala/sbt/util/ActionCacheStore.scala @@ -190,7 +190,10 @@ class DiskActionCacheStore(base: Path) extends ActionCacheStore: refs.flatMap: r => val casFile = casBase.toFile / r.contentHash if casFile.exists then - val outPath = outputDirectory.resolve(r.id) + val id = + if r.id.startsWith("${OUT}/") then r.id.drop(7) + else r.id + val outPath = outputDirectory.resolve(id) Files.createDirectories(outPath.getParent()) if outPath.toFile().exists() then IO.delete(outPath.toFile()) Some(Files.createSymbolicLink(outPath, casFile.toPath)) diff --git a/util-cache-resolver/src/main/scala/sbt/util/HashedVirtualFileRef.scala b/util-cache-resolver/src/main/scala/sbt/util/HashedVirtualFileRef.scala index 11164ce417..8d0b0cab73 100644 --- a/util-cache-resolver/src/main/scala/sbt/util/HashedVirtualFileRef.scala +++ b/util-cache-resolver/src/main/scala/sbt/util/HashedVirtualFileRef.scala @@ -1,5 +1,6 @@ package sbt.util +import xsbti.VirtualFile import xsbti.VirtualFileRef import sjsonnew.* @@ -26,6 +27,9 @@ object HashedVirtualFileRef: if xs.size != 2 then sys.error(s"invalid HashedVirtualFileRef $str") else HashedVirtualFileRef(xs(1), xs(0)) + def of(vf: VirtualFile): HashedVirtualFileRef = + val contentHash = s"farm64-${vf.contentHash.toHexString}" + HashedVirtualFileRef(vf.id, contentHash) given IsoLList.Aux[HashedVirtualFileRef, String :*: String :*: LNil] = LList.iso( { (r: HashedVirtualFileRef) => diff --git a/util-cache/src/main/scala/sbt/util/BasicCacheImplicits.scala b/util-cache/src/main/scala/sbt/util/BasicCacheImplicits.scala index d5f1114240..dc4b586cfd 100644 --- a/util-cache/src/main/scala/sbt/util/BasicCacheImplicits.scala +++ b/util-cache/src/main/scala/sbt/util/BasicCacheImplicits.scala @@ -26,5 +26,4 @@ trait BasicCacheImplicits: SingletonCache.basicSingletonCache(asSingleton(t)) given IsoString[VirtualFileRef] = IsoString.iso(_.id, VirtualFileRef.of) - given JsonFormat[VirtualFileRef] = summon[JsonFormat[VirtualFileRef]] end BasicCacheImplicits