Skip to content

Commit

Permalink
Fix #8634: Support -release option
Browse files Browse the repository at this point in the history 
* A port of https://github.com/scala/scala/pull/6362/files with some improvements
* When running scalac on JDK 9+ the -release option assures that code is compiled with classes specific to the release available on the classpath.
  This applies to classes from the JDK itself and from external jars.
  If the compilation succeeds, bytecode for the specified release is produced.
* -target option gets renamed to -Xtarget. Using -release instead is preferred since -Xtarget sets the bytecode version without any checks so this might lead to producing code that breaks at runime
  • Loading branch information
@prolativ
prolativ committed Dec 10, 2020
1 parent a34130f commit 99f830e
Show file tree
Hide file tree
Showing 15 changed files with 531 additions and 86 deletions.
17 changes: 11 additions & 6 deletions compiler/src/dotty/tools/backend/jvm/BCodeIdiomatic.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,17 @@ trait BCodeIdiomatic {
import bTypes._
import coreBTypes._

lazy val classfileVersion: Int = ctx.settings.target.value match {
case "jvm-1.5" => asm.Opcodes.V1_5
case "jvm-1.6" => asm.Opcodes.V1_6
case "jvm-1.7" => asm.Opcodes.V1_7
case "jvm-1.8" => asm.Opcodes.V1_8
case "jvm-9" => asm.Opcodes.V9
lazy val target = Option(ctx.settings.release.value).filter(_.nonEmpty).getOrElse(ctx.settings.Xtarget.value)

lazy val classfileVersion: Int = target match {
case "8" => asm.Opcodes.V1_8
case "9" => asm.Opcodes.V9
case "10" => asm.Opcodes.V10
case "11" => asm.Opcodes.V11
case "12" => asm.Opcodes.V12
case "13" => asm.Opcodes.V13
case "14" => asm.Opcodes.V14
case "15" => asm.Opcodes.V15
}

lazy val majorVersion: Int = (classfileVersion & 0xFF)
Expand Down
133 changes: 102 additions & 31 deletions compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@ import java.io.{File => JFile}
import java.net.URL
import java.nio.file.{FileSystems, Files}

import dotty.tools.io.{AbstractFile, PlainFile, ClassPath, ClassRepresentation, EfficientClassPath}
import dotty.tools.dotc.classpath.PackageNameUtils.{packageContains, separatePkgAndClassNames}
import dotty.tools.io.{AbstractFile, PlainFile, ClassPath, ClassRepresentation, EfficientClassPath, JDK9Reflectors}
import FileUtils._
import PlainFile.toPlainFile

import scala.collection.JavaConverters._
import scala.collection.immutable.ArraySeq
import scala.util.control.NonFatal

/**
* A trait allowing to look for classpath entries in directories. It provides common logic for
Expand Down Expand Up @@ -111,7 +114,7 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo
else Array()
}
protected def getName(f: JFile): String = f.getName
protected def toAbstractFile(f: JFile): AbstractFile = new PlainFile(new dotty.tools.io.File(f.toPath))
protected def toAbstractFile(f: JFile): AbstractFile = f.toPath.toPlainFile
protected def isPackage(f: JFile): Boolean = f.isPackage

assert(dir != null, "Directory file in DirectoryFileLookup cannot be null")
Expand All @@ -122,15 +125,33 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo

object JrtClassPath {
import java.nio.file._, java.net.URI
def apply(): Option[ClassPath] =
try {
val fs = FileSystems.getFileSystem(URI.create("jrt:/"))
Some(new JrtClassPath(fs))
}
catch {
case _: ProviderNotFoundException | _: FileSystemNotFoundException =>
None
def apply(release: Option[String]): Option[ClassPath] = {
import scala.util.Properties._
if (!isJavaAtLeast("9")) None
else {
// Longer term we'd like an official API for this in the JDK
// Discussion: http://mail.openjdk.java.net/pipermail/compiler-dev/2018-March/thread.html#11738

val currentMajorVersion: Int = JDK9Reflectors.runtimeVersionMajor(JDK9Reflectors.runtimeVersion()).intValue()
release match {
case Some(v) if v.toInt < currentMajorVersion =>
try {
val ctSym = Paths.get(javaHome).resolve("lib").resolve("ct.sym")
if (Files.notExists(ctSym)) None
else Some(new CtSymClassPath(ctSym, v.toInt))
} catch {
case NonFatal(_) => None
}
case _ =>
try {
val fs = FileSystems.getFileSystem(URI.create("jrt:/"))
Some(new JrtClassPath(fs))
} catch {
case _: ProviderNotFoundException | _: FileSystemNotFoundException => None
}
}
}
}
}

/**
Expand All @@ -157,20 +178,15 @@ final class JrtClassPath(fs: java.nio.file.FileSystem) extends ClassPath with No
/** Empty string represents root package */
override private[dotty] def hasPackage(pkg: PackageName): Boolean = packageToModuleBases.contains(pkg.dottedString)

override private[dotty] def packages(inPackage: PackageName): Seq[PackageEntry] = {
def matches(packageDottedName: String) =
if (packageDottedName.contains("."))
packageOf(packageDottedName) == inPackage.dottedString
else inPackage.isRoot
packageToModuleBases.keysIterator.filter(matches).map(PackageEntryImpl(_)).toVector
}
override private[dotty] def packages(inPackage: PackageName): Seq[PackageEntry] =
packageToModuleBases.keysIterator.filter(pack => packageContains(inPackage.dottedString, pack)).map(PackageEntryImpl(_)).toVector

private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] =
if (inPackage.isRoot) Nil
else
packageToModuleBases.getOrElse(inPackage.dottedString, Nil).flatMap(x =>
Files.list(x.resolve(inPackage.dirPathTrailingSlash)).iterator().asScala.filter(_.getFileName.toString.endsWith(".class"))).map(x =>
ClassFileEntryImpl(new PlainFile(new dotty.tools.io.File(x)))).toVector
ClassFileEntryImpl(x.toPlainFile)).toVector

override private[dotty] def list(inPackage: PackageName): ClassPathEntries =
if (inPackage.isRoot) ClassPathEntries(packages(inPackage), Nil)
Expand All @@ -184,14 +200,75 @@ final class JrtClassPath(fs: java.nio.file.FileSystem) extends ClassPath with No
def findClassFile(className: String): Option[AbstractFile] =
if (!className.contains(".")) None
else {
val inPackage = packageOf(className)
packageToModuleBases.getOrElse(inPackage, Nil).iterator.flatMap{x =>
val (inPackage, _) = separatePkgAndClassNames(className)
packageToModuleBases.getOrElse(inPackage, Nil).iterator.flatMap{ x =>
val file = x.resolve(FileUtils.dirPath(className) + ".class")
if (Files.exists(file)) new PlainFile(new dotty.tools.io.File(file)) :: Nil else Nil
if (Files.exists(file)) file.toPlainFile :: Nil else Nil
}.take(1).toList.headOption
}
private def packageOf(dottedClassName: String): String =
dottedClassName.substring(0, dottedClassName.lastIndexOf("."))
}

/**
* Implementation `ClassPath` based on the \$JAVA_HOME/lib/ct.sym backing http://openjdk.java.net/jeps/247
*/
final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths {
import java.nio.file.Path, java.nio.file._

private val fileSystem: FileSystem = FileSystems.newFileSystem(ctSym, null: ClassLoader)
private val root: Path = fileSystem.getRootDirectories.iterator.next
private val roots = Files.newDirectoryStream(root).iterator.asScala.toList

// http://mail.openjdk.java.net/pipermail/compiler-dev/2018-March/011737.html
private def codeFor(major: Int): String = if (major < 10) major.toString else ('A' + (major - 10)).toChar.toString

private val releaseCode: String = codeFor(release)
private def fileNameMatchesRelease(fileName: String) = !fileName.contains("-") && fileName.contains(releaseCode) // exclude `9-modules`
private val rootsForRelease: List[Path] = roots.filter(root => fileNameMatchesRelease(root.getFileName.toString))

// e.g. "java.lang" -> Seq(/876/java/lang, /87/java/lang, /8/java/lang))
private val packageIndex: scala.collection.Map[String, scala.collection.Seq[Path]] = {
val index = collection.mutable.AnyRefMap[String, collection.mutable.ListBuffer[Path]]()
val isJava12OrHigher = scala.util.Properties.isJavaAtLeast("12")
rootsForRelease.foreach(root => Files.walk(root).iterator().asScala.filter(Files.isDirectory(_)).foreach { p =>
val moduleNamePathElementCount = if (isJava12OrHigher) 1 else 0
if (p.getNameCount > root.getNameCount + moduleNamePathElementCount) {
val packageDotted = p.subpath(moduleNamePathElementCount + root.getNameCount, p.getNameCount).toString.replace('/', '.')
index.getOrElseUpdate(packageDotted, new collection.mutable.ListBuffer) += p
}
})
index
}

/** Empty string represents root package */
override private[dotty] def hasPackage(pkg: PackageName) = packageIndex.contains(pkg.dottedString)
override private[dotty] def packages(inPackage: PackageName): Seq[PackageEntry] = {
packageIndex.keysIterator.filter(pack => packageContains(inPackage.dottedString, pack)).map(PackageEntryImpl(_)).toVector
}
private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = {
if (inPackage.isRoot) Nil
else {
val sigFiles = packageIndex.getOrElse(inPackage.dottedString, Nil).iterator.flatMap(p =>
Files.list(p).iterator.asScala.filter(_.getFileName.toString.endsWith(".sig")))
sigFiles.map(f => ClassFileEntryImpl(f.toPlainFile)).toVector
}
}

override private[dotty] def list(inPackage: PackageName): ClassPathEntries =
if (inPackage.isRoot) ClassPathEntries(packages(inPackage), Nil)
else ClassPathEntries(packages(inPackage), classes(inPackage))

def asURLs: Seq[URL] = Nil
def asClassPathStrings: Seq[String] = Nil
def findClassFile(className: String): Option[AbstractFile] = {
if (!className.contains(".")) None
else {
val (inPackage, classSimpleName) = separatePkgAndClassNames(className)
packageIndex.getOrElse(inPackage, Nil).iterator.flatMap { p =>
val path = p.resolve(classSimpleName + ".sig")
if (Files.exists(path)) path.toPlainFile :: Nil else Nil
}.take(1).toList.headOption
}
}
}

case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFileEntryImpl] with NoSourcePaths {
Expand All @@ -201,9 +278,7 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFile
val relativePath = FileUtils.dirPath(className)
val classFile = new JFile(dir, relativePath + ".class")
if (classFile.exists) {
val wrappedClassFile = new dotty.tools.io.File(classFile.toPath)
val abstractClassFile = new PlainFile(wrappedClassFile)
Some(abstractClassFile)
Some(classFile.toPath.toPlainFile)
}
else None
}
Expand All @@ -228,11 +303,7 @@ case class DirectorySourcePath(dir: JFile) extends JFileDirectoryLookup[SourceFi
.map(ext => new JFile(dir, relativePath + "." + ext))
.collectFirst { case file if file.exists() => file }

sourceFile.map { file =>
val wrappedSourceFile = new dotty.tools.io.File(file.toPath)
val abstractSourceFile = new PlainFile(wrappedSourceFile)
abstractSourceFile
}
sourceFile.map(_.toPath.toPlainFile)
}

private[dotty] def sources(inPackage: PackageName): Seq[SourceFileEntry] = files(inPackage)
Expand Down
17 changes: 12 additions & 5 deletions compiler/src/dotty/tools/dotc/classpath/FileUtils.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ object FileUtils {
// FIXME: drop last condition when we stop being compatible with Scala 2.11
}

private val SUFFIX_CLASS = ".class"
private val SUFFIX_SCALA = ".scala"
private val SUFFIX_JAVA = ".java"
private val SUFFIX_SIG = ".sig"

def stripSourceExtension(fileName: String): String =
if (endsScala(fileName)) stripClassExtension(fileName)
else if (endsJava(fileName)) stripJavaExtension(fileName)
Expand All @@ -46,23 +51,25 @@ object FileUtils {

def dirPathInJar(forPackage: String): String = forPackage.replace('.', '/')

inline private def ends (filename:String, suffix:String) = filename.endsWith(suffix) && filename.length > suffix.length

def endsClass(fileName: String): Boolean =
fileName.length > 6 && fileName.substring(fileName.length - 6) == ".class"
ends (fileName, SUFFIX_CLASS) || fileName.endsWith(SUFFIX_SIG)

def endsScalaOrJava(fileName: String): Boolean =
endsScala(fileName) || endsJava(fileName)

def endsJava(fileName: String): Boolean =
fileName.length > 5 && fileName.substring(fileName.length - 5) == ".java"
ends (fileName, SUFFIX_JAVA)

def endsScala(fileName: String): Boolean =
fileName.length > 6 && fileName.substring(fileName.length - 6) == ".scala"
ends (fileName, SUFFIX_SCALA)

def stripClassExtension(fileName: String): String =
fileName.substring(0, fileName.length - 6) // equivalent of fileName.length - ".class".length
fileName.substring(0, fileName.lastIndexOf('.'))

def stripJavaExtension(fileName: String): String =
fileName.substring(0, fileName.length - 5)
fileName.substring(0, fileName.length - 5) // equivalent of fileName.length - SUFFIX_JAVA.length

// probably it should match a pattern like [a-z_]{1}[a-z0-9_]* but it cannot be changed
// because then some tests in partest don't pass
Expand Down
13 changes: 12 additions & 1 deletion compiler/src/dotty/tools/dotc/classpath/PackageNameUtils.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ object PackageNameUtils {
* @param fullClassName full class name with package
* @return (package, simple class name)
*/
def separatePkgAndClassNames(fullClassName: String): (String, String) = {
inline def separatePkgAndClassNames(fullClassName: String): (String, String) = {
val lastDotIndex = fullClassName.lastIndexOf('.')
if (lastDotIndex == -1)
(RootPackage, fullClassName)
Expand All @@ -23,4 +23,15 @@ object PackageNameUtils {
}

def packagePrefix(inPackage: String): String = if (inPackage == RootPackage) "" else inPackage + "."

/**
* `true` if `packageDottedName` is a package directly nested in `inPackage`, for example:
* - `packageContains("scala", "scala.collection")`
* - `packageContains("", "scala")`
*/
def packageContains(inPackage: String, packageDottedName: String) = {
if (packageDottedName.contains("."))
packageDottedName.startsWith(inPackage) && packageDottedName.lastIndexOf('.') == inPackage.length
else inPackage == ""
}
}
21 changes: 12 additions & 9 deletions compiler/src/dotty/tools/dotc/classpath/ZipAndJarFileLookupFactory.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,22 @@ sealed trait ZipAndJarFileLookupFactory {
private val cache = new FileBasedCache[ClassPath]

def create(zipFile: AbstractFile)(using Context): ClassPath =
if (ctx.settings.YdisableFlatCpCaching.value || zipFile.file == null) createForZipFile(zipFile)
else createUsingCache(zipFile)
val release = Option(ctx.settings.release.value).filter(_.nonEmpty)
if (ctx.settings.YdisableFlatCpCaching.value || zipFile.file == null) createForZipFile(zipFile, release)
else createUsingCache(zipFile, release)

protected def createForZipFile(zipFile: AbstractFile): ClassPath
protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath

private def createUsingCache(zipFile: AbstractFile): ClassPath =
cache.getOrCreate(zipFile.file.toPath, () => createForZipFile(zipFile))
private def createUsingCache(zipFile: AbstractFile, release: Option[String]): ClassPath =
cache.getOrCreate(zipFile.file.toPath, () => createForZipFile(zipFile, release))
}

/**
* Manages creation of classpath for class files placed in zip and jar files.
* It should be the only way of creating them as it provides caching.
*/
object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
private case class ZipArchiveClassPath(zipFile: File)
private case class ZipArchiveClassPath(zipFile: File, override val release: Option[String])
extends ZipArchiveFileLookup[ClassFileEntryImpl]
with NoSourcePaths {

Expand Down Expand Up @@ -141,9 +142,9 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
case class PackageInfo(packageName: String, subpackages: List[AbstractFile])
}

override protected def createForZipFile(zipFile: AbstractFile): ClassPath =
override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath =
if (zipFile.file == null) createWithoutUnderlyingFile(zipFile)
else ZipArchiveClassPath(zipFile.file)
else ZipArchiveClassPath(zipFile.file, release)

private def createWithoutUnderlyingFile(zipFile: AbstractFile) = zipFile match {
case manifestRes: ManifestResources =>
Expand All @@ -162,6 +163,8 @@ object ZipAndJarSourcePathFactory extends ZipAndJarFileLookupFactory {
private case class ZipArchiveSourcePath(zipFile: File)
extends ZipArchiveFileLookup[SourceFileEntryImpl]
with NoClassPaths {

def release: Option[String] = None

override def asSourcePathString: String = asClassPathString

Expand All @@ -171,7 +174,7 @@ object ZipAndJarSourcePathFactory extends ZipAndJarFileLookupFactory {
override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isScalaOrJavaSource
}

override protected def createForZipFile(zipFile: AbstractFile): ClassPath = ZipArchiveSourcePath(zipFile.file)
override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath = ZipArchiveSourcePath(zipFile.file)
}

final class FileBasedCache[T] {
Expand Down
3 changes: 2 additions & 1 deletion compiler/src/dotty/tools/dotc/classpath/ZipArchiveFileLookup.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@ import dotty.tools.io.{EfficientClassPath, ClassRepresentation}
*/
trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends EfficientClassPath {
val zipFile: File
def release: Option[String]

assert(zipFile != null, "Zip file in ZipArchiveFileLookup cannot be null")

override def asURLs: Seq[URL] = Seq(zipFile.toURI.toURL)
override def asClassPathStrings: Seq[String] = Seq(zipFile.getPath)

private val archive = new FileZipArchive(zipFile.toPath)
private val archive = new FileZipArchive(zipFile.toPath, release)

override private[dotty] def packages(inPackage: PackageName): Seq[PackageEntry] = {
for {
Expand Down
23 changes: 13 additions & 10 deletions compiler/src/dotty/tools/dotc/config/PathResolver.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,16 +206,19 @@ class PathResolver(using c: Context) {
import classPathFactory._

// Assemble the elements!
def basis: List[Traversable[ClassPath]] = List(
JrtClassPath.apply(), // 1. The Java 9 classpath (backed by the jrt:/ virtual system, if available)
classesInPath(javaBootClassPath), // 2. The Java bootstrap class path.
contentsOfDirsInPath(javaExtDirs), // 3. The Java extension class path.
classesInExpandedPath(javaUserClassPath), // 4. The Java application class path.
classesInPath(scalaBootClassPath), // 5. The Scala boot class path.
contentsOfDirsInPath(scalaExtDirs), // 6. The Scala extension class path.
classesInExpandedPath(userClassPath), // 7. The Scala application class path.
sourcesInPath(sourcePath) // 8. The Scala source path.
)
def basis: List[Traversable[ClassPath]] =
val release = Option(ctx.settings.release.value).filter(_.nonEmpty)

List(
JrtClassPath(release), // 1. The Java 9+ classpath (backed by the jrt:/ virtual system, if available)
classesInPath(javaBootClassPath), // 2. The Java bootstrap class path.
contentsOfDirsInPath(javaExtDirs), // 3. The Java extension class path.
classesInExpandedPath(javaUserClassPath), // 4. The Java application class path.
classesInPath(scalaBootClassPath), // 5. The Scala boot class path.
contentsOfDirsInPath(scalaExtDirs), // 6. The Scala extension class path.
classesInExpandedPath(userClassPath), // 7. The Scala application class path.
sourcesInPath(sourcePath) // 8. The Scala source path.
)

lazy val containers: List[ClassPath] = basis.flatten.distinct

Expand Down
Loading

0 comments on commit 99f830e

Please sign in to comment.