From 7e69d56c53975f2eedf090904018b5b073876b58 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Wed, 10 Jul 2024 23:16:47 +0200 Subject: [PATCH 1/3] Fix notify Zinc about class file in jar [Cherry-picked 6d7f782bef6f2614f227d2416110419dc36481ab][modified] --- .../tools/backend/jvm/ClassfileWriters.scala | 27 +++++++++++-------- .../tools/backend/jvm/PostProcessor.scala | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala b/compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala index 928d19c92041..671ba0b3088f 100644 --- a/compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala +++ b/compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala @@ -1,6 +1,6 @@ package dotty.tools.backend.jvm -import java.io.{DataOutputStream, IOException, BufferedOutputStream, FileOutputStream} +import java.io.{DataOutputStream, File, IOException, BufferedOutputStream, FileOutputStream} import java.nio.ByteBuffer import java.nio.channels.{ClosedByInterruptException, FileChannel} import java.nio.charset.StandardCharsets.UTF_8 @@ -12,7 +12,7 @@ import java.util.zip.{CRC32, Deflater, ZipEntry, ZipOutputStream} import dotty.tools.dotc.core.Contexts.* import dotty.tools.dotc.core.Decorators.em -import dotty.tools.io.{AbstractFile, PlainFile} +import dotty.tools.io.{AbstractFile, PlainFile, VirtualFile} import dotty.tools.io.PlainFile.toPlainFile import BTypes.InternalName import scala.util.chaining._ @@ -22,7 +22,6 @@ import scala.language.unsafeNulls class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) { - type NullableFile = AbstractFile | Null import frontendAccess.{compilerSettings, backendReporting} sealed trait TastyWriter { @@ -42,7 +41,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) { /** * Write a classfile */ - def writeClass(name: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): NullableFile + def writeClass(name: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile /** @@ -87,7 +86,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) { } private final class SingleClassWriter(underlying: FileWriter) extends ClassfileWriter { - override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): NullableFile = { + override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile = { underlying.writeFile(classRelativePath(className), bytes) } override def writeTasty(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): Unit = { @@ -99,7 +98,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) { } private final class DebugClassWriter(basic: ClassfileWriter, dump: FileWriter) extends ClassfileWriter { - override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): NullableFile = { + override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile = { val outFile = basic.writeClass(className, bytes, sourceFile) dump.writeFile(classRelativePath(className), bytes) outFile @@ -117,7 +116,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) { } sealed trait FileWriter { - def writeFile(relativePath: String, bytes: Array[Byte]): NullableFile + def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile def close(): Unit } @@ -161,7 +160,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) { lazy val crc = new CRC32 - override def writeFile(relativePath: String, bytes: Array[Byte]): NullableFile = this.synchronized { + override def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile = this.synchronized { val entry = new ZipEntry(relativePath) if (storeOnly) { // When using compression method `STORED`, the ZIP spec requires the CRC and compressed/ @@ -178,7 +177,13 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) { jarWriter.putNextEntry(entry) try jarWriter.write(bytes, 0, bytes.length) finally jarWriter.flush() - null + // important detail here, even on Windows, Zinc expects the separator within the jar + // to be the system default, (even if in the actual jar file the entry always uses '/'). + // see https://github.com/sbt/zinc/blob/dcddc1f9cfe542d738582c43f4840e17c053ce81/internal/compiler-bridge/src/main/scala/xsbt/JarUtils.scala#L47 + val pathInJar = + if File.separatorChar == '/' then relativePath + else relativePath.replace('/', File.separatorChar) + PlainFile.toPlainFile(Paths.get(s"${file.absolutePath}!$pathInJar")) } override def close(): Unit = this.synchronized(jarWriter.close()) @@ -226,7 +231,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) { private val fastOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE) private val fallbackOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING) - override def writeFile(relativePath: String, bytes: Array[Byte]): NullableFile = { + override def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile = { val path = base.resolve(relativePath) try { ensureDirForPath(base, path) @@ -275,7 +280,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) { finally out.close() } - override def writeFile(relativePath: String, bytes: Array[Byte]):NullableFile = { + override def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile = { val outFile = getFile(base, relativePath) writeBytes(outFile, bytes) outFile diff --git a/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala b/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala index 5445c4a83b96..55d2f4aa3bce 100644 --- a/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala +++ b/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala @@ -48,7 +48,7 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes: if AsmUtils.traceSerializedClassEnabled && internalName.contains(AsmUtils.traceSerializedClassPattern) then AsmUtils.traceClass(bytes) val clsFile = classfileWriter.writeClass(internalName, bytes, sourceFile) - if clsFile != null then clazz.onFileCreated(clsFile) + clazz.onFileCreated(clsFile) } def sendToDisk(tasty: GeneratedTasty, sourceFile: AbstractFile): Unit = { From 6e08873656cbf74f4da8b87b01da268619534f8b Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Wed, 10 Jul 2024 23:20:06 +0200 Subject: [PATCH 2/3] Fix toString of JarArchive It returns the absolute path of the jar file instead of '/' [Cherry-picked 2fb98fd36b02bc30a32545e6ef07d979afdb39f9][modified] --- compiler/src/dotty/tools/io/JarArchive.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/compiler/src/dotty/tools/io/JarArchive.scala b/compiler/src/dotty/tools/io/JarArchive.scala index 49b743e83074..014768640ea7 100644 --- a/compiler/src/dotty/tools/io/JarArchive.scala +++ b/compiler/src/dotty/tools/io/JarArchive.scala @@ -10,10 +10,12 @@ import scala.jdk.CollectionConverters.* * This class implements an [[AbstractFile]] backed by a jar * that be can used as the compiler's output directory. */ -class JarArchive private (root: Directory) extends PlainDirectory(root) { - def close(): Unit = jpath.getFileSystem().close() +class JarArchive private (val jarPath: Path, root: Directory) extends PlainDirectory(root) { + def close(): Unit = this.synchronized(jpath.getFileSystem().close()) def allFileNames(): Iterator[String] = java.nio.file.Files.walk(jpath).iterator().asScala.map(_.toString) + + override def toString: String = jarPath.toString } object JarArchive { @@ -39,6 +41,6 @@ object JarArchive { } } val root = fs.getRootDirectories().iterator.next() - new JarArchive(Directory(root)) + new JarArchive(path, Directory(root)) } } From 6e76dbeb9af04fb843313fc76c58a7cb56b8df6b Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Wed, 10 Jul 2024 23:28:08 +0200 Subject: [PATCH 3/3] Test products of compilation to jar [Cherry-picked 80076269693bfb7bb4639c9898220ce47b966d87][modified] --- .../xsbt/ExtractUsedNamesSpecification.scala | 5 +- .../test/xsbt/ProductsSpecification.scala | 34 ++++++ .../xsbt/ScalaCompilerForUnitTesting.scala | 102 +++++++++--------- 3 files changed, 89 insertions(+), 52 deletions(-) create mode 100644 sbt-bridge/test/xsbt/ProductsSpecification.scala diff --git a/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala b/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala index e47371175de6..0abefe2985c3 100644 --- a/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala +++ b/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala @@ -1,7 +1,6 @@ package xsbt import xsbti.UseScope -import ScalaCompilerForUnitTesting.Callbacks import org.junit.{ Test, Ignore } import org.junit.Assert._ @@ -227,9 +226,9 @@ class ExtractUsedNamesSpecification { def findPatMatUsages(in: String): Set[String] = { val compilerForTesting = new ScalaCompilerForUnitTesting - val (_, Callbacks(callback, _)) = + val output = compilerForTesting.compileSrcs(List(List(sealedClass, in))) - val clientNames = callback.usedNamesAndScopes.view.filterKeys(!_.startsWith("base.")) + val clientNames = output.analysis.usedNamesAndScopes.view.filterKeys(!_.startsWith("base.")) val names: Set[String] = clientNames.flatMap { case (_, usages) => diff --git a/sbt-bridge/test/xsbt/ProductsSpecification.scala b/sbt-bridge/test/xsbt/ProductsSpecification.scala new file mode 100644 index 000000000000..b13defecc4cc --- /dev/null +++ b/sbt-bridge/test/xsbt/ProductsSpecification.scala @@ -0,0 +1,34 @@ +package xsbt + +import org.junit.Assert.* +import org.junit.Ignore +import org.junit.Test + +import java.io.File +import java.nio.file.Path +import java.nio.file.Paths + +class ProductsSpecification { + + @Test + def extractProductsFromJar = { + val src = + """package example + | + |class A { + | class B + | def foo = + | class C + |}""".stripMargin + val output = compiler.compileSrcsToJar(src) + val srcFile = output.srcFiles.head + val products = output.analysis.productClassesToSources.filter(_._2 == srcFile).keys.toSet + + def toPathInJar(className: String): Path = + Paths.get(s"${output.classesOutput}!${className.replace('.', File.separatorChar)}.class") + val expected = Set("example.A", "example.A$B", "example.A$C$1").map(toPathInJar) + assertEquals(products, expected) + } + + private def compiler = new ScalaCompilerForUnitTesting +} \ No newline at end of file diff --git a/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala b/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala index 87bc45744e21..bd6ec2b8d63e 100644 --- a/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala +++ b/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala @@ -1,22 +1,19 @@ /** Adapted from https://github.com/sbt/sbt/blob/0.13/compile/interface/src/test/scala/xsbt/ScalaCompilerForUnitTesting.scala */ package xsbt -import xsbti.compile.{CompileProgress, SingleOutput} -import java.io.File -import xsbti._ -import sbt.io.IO -import xsbti.api.{ ClassLike, Def, DependencyContext } -import DependencyContext._ -import xsbt.api.SameAPI -import sbt.internal.util.ConsoleLogger -import dotty.tools.io.PlainFile.toPlainFile import dotty.tools.xsbt.CompilerBridge +import sbt.io.IO +import xsbti.* +import xsbti.api.ClassLike +import xsbti.api.DependencyContext.* +import xsbti.compile.SingleOutput + +import java.io.File +import java.nio.file.Path import TestCallback.ExtractedClassDependencies -import ScalaCompilerForUnitTesting.Callbacks -object ScalaCompilerForUnitTesting: - case class Callbacks(analysis: TestCallback, progress: TestCompileProgress) +case class CompileOutput(srcFiles: Seq[VirtualFileRef], classesOutput: Path, analysis: TestCallback, progress: TestCompileProgress) /** * Provides common functionality needed for unit tests that require compiling @@ -25,29 +22,24 @@ object ScalaCompilerForUnitTesting: class ScalaCompilerForUnitTesting { def extractEnteredPhases(srcs: String*): Seq[List[String]] = { - val (tempSrcFiles, Callbacks(_, testProgress)) = compileSrcs(srcs: _*) - val run = testProgress.runs.head - tempSrcFiles.map(src => run.unitPhases(src.id)) + val output = compileSrcs(srcs*) + val run = output.progress.runs.head + output.srcFiles.map(src => run.unitPhases(src.id)) } - def extractTotal(srcs: String*)(extraSourcePath: String*): Int = { - val (tempSrcFiles, Callbacks(_, testProgress)) = compileSrcs(List(srcs.toList), extraSourcePath.toList) - val run = testProgress.runs.head - run.total - } + def extractTotal(srcs: String*)(extraSourcePath: String*): Int = + compileSrcs(List(srcs.toList), extraSourcePath.toList).progress.runs.head.total - def extractProgressPhases(srcs: String*): List[String] = { - val (_, Callbacks(_, testProgress)) = compileSrcs(srcs: _*) - testProgress.runs.head.phases - } + def extractProgressPhases(srcs: String*): List[String] = + compileSrcs(srcs*).progress.runs.head.phases /** * Compiles given source code using Scala compiler and returns API representation * extracted by ExtractAPI class. */ def extractApiFromSrc(src: String): Seq[ClassLike] = { - val (Seq(tempSrcFile), Callbacks(analysisCallback, _)) = compileSrcs(src) - analysisCallback.apis(tempSrcFile) + val output = compileSrcs(src) + output.analysis.apis(output.srcFiles.head) } /** @@ -55,8 +47,8 @@ class ScalaCompilerForUnitTesting { * extracted by ExtractAPI class. */ def extractApisFromSrcs(srcs: List[String]*): Seq[Seq[ClassLike]] = { - val (tempSrcFiles, Callbacks(analysisCallback, _)) = compileSrcs(srcs.toList) - tempSrcFiles.map(analysisCallback.apis) + val output = compileSrcs(srcs.toList) + output.srcFiles.map(output.analysis.apis) } /** @@ -73,15 +65,16 @@ class ScalaCompilerForUnitTesting { assertDefaultScope: Boolean = true ): Map[String, Set[String]] = { // we drop temp src file corresponding to the definition src file - val (Seq(_, tempSrcFile), Callbacks(analysisCallback, _)) = compileSrcs(definitionSrc, actualSrc) + val output = compileSrcs(definitionSrc, actualSrc) + val analysis = output.analysis if (assertDefaultScope) for { - (className, used) <- analysisCallback.usedNamesAndScopes - analysisCallback.TestUsedName(name, scopes) <- used + (className, used) <- analysis.usedNamesAndScopes + analysis.TestUsedName(name, scopes) <- used } assert(scopes.size() == 1 && scopes.contains(UseScope.Default), s"$className uses $name in $scopes") - val classesInActualSrc = analysisCallback.classNames(tempSrcFile).map(_._1) - classesInActualSrc.map(className => className -> analysisCallback.usedNames(className)).toMap + val classesInActualSrc = analysis.classNames(output.srcFiles.head).map(_._1) + classesInActualSrc.map(className => className -> analysis.usedNames(className)).toMap } /** @@ -91,11 +84,11 @@ class ScalaCompilerForUnitTesting { * Only the names used in the last src file are returned. */ def extractUsedNamesFromSrc(sources: String*): Map[String, Set[String]] = { - val (srcFiles, Callbacks(analysisCallback, _)) = compileSrcs(sources: _*) - srcFiles + val output = compileSrcs(sources*) + output.srcFiles .map { srcFile => - val classesInSrc = analysisCallback.classNames(srcFile).map(_._1) - classesInSrc.map(className => className -> analysisCallback.usedNames(className)).toMap + val classesInSrc = output.analysis.classNames(srcFile).map(_._1) + classesInSrc.map(className => className -> output.analysis.usedNames(className)).toMap } .reduce(_ ++ _) } @@ -113,15 +106,15 @@ class ScalaCompilerForUnitTesting { * file system-independent way of testing dependencies between source code "files". */ def extractDependenciesFromSrcs(srcs: List[List[String]]): ExtractedClassDependencies = { - val (_, Callbacks(testCallback, _)) = compileSrcs(srcs) + val analysis = compileSrcs(srcs).analysis - val memberRefDeps = testCallback.classDependencies collect { + val memberRefDeps = analysis.classDependencies collect { case (target, src, DependencyByMemberRef) => (src, target) } - val inheritanceDeps = testCallback.classDependencies collect { + val inheritanceDeps = analysis.classDependencies collect { case (target, src, DependencyByInheritance) => (src, target) } - val localInheritanceDeps = testCallback.classDependencies collect { + val localInheritanceDeps = analysis.classDependencies collect { case (target, src, LocalDependencyByInheritance) => (src, target) } ExtractedClassDependencies.fromPairs(memberRefDeps, inheritanceDeps, localInheritanceDeps) @@ -142,12 +135,20 @@ class ScalaCompilerForUnitTesting { * The sequence of temporary files corresponding to passed snippets and analysis * callback is returned as a result. */ - def compileSrcs(groupedSrcs: List[List[String]], sourcePath: List[String] = Nil): (Seq[VirtualFile], Callbacks) = { + def compileSrcs(groupedSrcs: List[List[String]], sourcePath: List[String] = Nil, compileToJar: Boolean = false): CompileOutput = { val temp = IO.createTemporaryDirectory val analysisCallback = new TestCallback val testProgress = new TestCompileProgress - val classesDir = new File(temp, "classes") - classesDir.mkdir() + val classesOutput = + if (compileToJar) { + val jar = new File(temp, "classes.jar") + jar.createNewFile() + jar + } else { + val dir = new File(temp, "classes") + dir.mkdir() + dir + } val bridge = new CompilerBridge @@ -164,16 +165,16 @@ class ScalaCompilerForUnitTesting { } val virtualSrcFiles = srcFiles.toArray - val classesDirPath = classesDir.getAbsolutePath.toString + val classesOutputPath = classesOutput.getAbsolutePath() val output = new SingleOutput: - def getOutputDirectory() = classesDir + def getOutputDirectory() = classesOutput val maybeSourcePath = if extraFiles.isEmpty then Nil else List("-sourcepath", temp.getAbsolutePath.toString) bridge.run( virtualSrcFiles, new TestDependencyChanges, - Array("-Yforce-sbt-phases", "-classpath", classesDirPath, "-usejavacp", "-d", classesDirPath) ++ maybeSourcePath, + Array("-Yforce-sbt-phases", "-classpath", classesOutputPath, "-usejavacp", "-d", classesOutputPath) ++ maybeSourcePath, output, analysisCallback, new TestReporter, @@ -185,13 +186,16 @@ class ScalaCompilerForUnitTesting { srcFiles } - (files.flatten.toSeq, Callbacks(analysisCallback, testProgress)) + CompileOutput(files.flatten.toSeq, classesOutput.toPath, analysisCallback, testProgress) } - def compileSrcs(srcs: String*): (Seq[VirtualFile], Callbacks) = { + def compileSrcs(srcs: String*): CompileOutput = { compileSrcs(List(srcs.toList)) } + def compileSrcsToJar(srcs: String*): CompileOutput = + compileSrcs(List(srcs.toList), compileToJar = true) + private def prepareSrcFile(baseDir: File, fileName: String, src: String): VirtualFile = { val srcFile = new File(baseDir, fileName) IO.write(srcFile, src)