Add Stable Presentation Compiler

NOTICE.md

@@ -89,15 +89,19 @@ major authors were omitted by oversight.
     details.
 
   * dotty.tools.dotc.coverage: Coverage instrumentation utilities have been
-    adapted from the scoverage plugin for scala 2 [5], which is under the
+    adapted from the scoverage plugin for scala 2 [4], which is under the
     Apache 2.0 license.
 
+  * dooty.tools.pc: Presentation compiler implementation adapted from
+    scalameta/metals [5] mtags module, which is under the Apache 2.0 license.
+
   * The Dotty codebase contains parts which are derived from
-    the ScalaPB protobuf library [4], which is under the Apache 2.0 license.
+    the ScalaPB protobuf library [6], which is under the Apache 2.0 license.
 
 
 [1] https://github.com/scala/scala
 [2] https://github.com/adriaanm/scala/tree/sbt-api-consolidate/src/compiler/scala/tools/sbt
 [3] https://github.com/sbt/sbt/tree/0.13/compile/interface/src/main/scala/xsbt
-[4] https://github.com/lampepfl/dotty/pull/5783/files
-[5] https://github.com/scoverage/scalac-scoverage-plugin
+[4] https://github.com/scoverage/scalac-scoverage-plugin
+[5] https://github.com/scalameta/metals
+[6] https://github.com/lampepfl/dotty/pull/5783/files

build.sbt

@@ -28,6 +28,8 @@ val `scala3-bench-run` = Build.`scala3-bench-run`
 val dist = Build.dist
 val `community-build` = Build.`community-build`
 val `sbt-community-build` = Build.`sbt-community-build`
+val `scala3-presentation-compiler` = Build.`scala3-presentation-compiler`
+val `scala3-presentation-compiler-bootstrapped` = Build.`scala3-presentation-compiler-bootstrapped`
 
 val sjsSandbox = Build.sjsSandbox
 val sjsJUnitTests = Build.sjsJUnitTests

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -5,6 +5,7 @@ import core._
 import Contexts._
 import SymDenotations.ClassDenotation
 import Symbols._
+import Comments.Comment
 import util.{FreshNameCreator, SourceFile, NoSource}
 import util.Spans.Span
 import ast.{tpd, untpd}
@@ -69,6 +70,9 @@ class CompilationUnit protected (val source: SourceFile) {
   /** Can this compilation unit be suspended */
   def isSuspendable: Boolean = true
 
+  /** List of all comments present in this compilation unit */
+  var comments: List[Comment] = Nil
+
   /** Suspends the compilation unit by thowing a SuspendException
    *  and recording the suspended compilation unit
    */

compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala

@@ -145,7 +145,7 @@ class InteractiveDriver(val settings: List[String]) extends Driver {
     (fromSource ++ fromClassPath).distinct
   }
 
-  def run(uri: URI, sourceCode: String): List[Diagnostic] = run(uri, toSource(uri, sourceCode))
+  def run(uri: URI, sourceCode: String): List[Diagnostic] = run(uri, SourceFile.virtual(uri, sourceCode))
 
   def run(uri: URI, source: SourceFile): List[Diagnostic] = {
     import typer.ImportInfo._
@@ -297,9 +297,6 @@ class InteractiveDriver(val settings: List[String]) extends Driver {
     cleanupTree(tree)
   }
 
-  private def toSource(uri: URI, sourceCode: String): SourceFile =
-    SourceFile.virtual(Paths.get(uri).toString, sourceCode)
-
   /**
    * Initialize this driver and compiler.
    *
@@ -323,7 +320,7 @@ object InteractiveDriver {
     else
       try
         // We don't use file.file here since it'll be null
-        // for the VirtualFiles created by InteractiveDriver#toSource
+        // for the VirtualFiles created by SourceFile#virtual
         // TODO: To avoid these round trip conversions, we could add an
         // AbstractFile#toUri method and implement it by returning a constant
         // passed as a parameter to a constructor of VirtualFile

compiler/src/dotty/tools/dotc/parsing/ParserPhase.scala

@@ -30,6 +30,7 @@ class Parser extends Phase {
         val p = new Parsers.Parser(unit.source)
         //  p.in.debugTokenStream = true
         val tree = p.parse()
+        ctx.compilationUnit.comments = p.in.comments
         if (p.firstXmlPos.exists && !firstXmlPos.exists)
           firstXmlPos = p.firstXmlPos
         tree

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -227,11 +227,11 @@ object Scanners {
       */
     private var docstringMap: SortedMap[Int, Comment] = SortedMap.empty
 
-    /* A Buffer for comment positions */
-    private val commentPosBuf = new mutable.ListBuffer[Span]
+    /* A Buffer for comments */
+    private val commentBuf = new mutable.ListBuffer[Comment]
 
-    /** Return a list of all the comment positions */
-    def commentSpans: List[Span] = commentPosBuf.toList
+    /** Return a list of all the comments */
+    def comments: List[Comment] = commentBuf.toList
 
     private def addComment(comment: Comment): Unit = {
       val lookahead = lookaheadReader()
@@ -246,7 +246,7 @@ object Scanners {
     def getDocComment(pos: Int): Option[Comment] = docstringMap.get(pos)
 
     /** A buffer for comments */
-    private val commentBuf = CharBuffer(initialCharBufferSize)
+    private val currentCommentBuf = CharBuffer(initialCharBufferSize)
 
     def toToken(identifier: SimpleName): Token =
       def handleMigration(keyword: Token): Token =
@@ -523,7 +523,7 @@ object Scanners {
      *
      *      The following tokens can start an indentation region:
      *
-     *         :  =  =>  <-  if  then  else  while  do  try  catch  
+     *         :  =  =>  <-  if  then  else  while  do  try  catch
      *         finally  for  yield  match  throw  return  with
      *
      *      Inserting an INDENT starts a new indentation region with the indentation of the current
@@ -1019,7 +1019,7 @@ object Scanners {
 
     private def skipComment(): Boolean = {
       def appendToComment(ch: Char) =
-        if (keepComments) commentBuf.append(ch)
+        if (keepComments) currentCommentBuf.append(ch)
       def nextChar() = {
         appendToComment(ch)
         Scanner.this.nextChar()
@@ -1047,9 +1047,9 @@ object Scanners {
       def finishComment(): Boolean = {
         if (keepComments) {
           val pos = Span(start, charOffset - 1, start)
-          val comment = Comment(pos, commentBuf.toString)
-          commentBuf.clear()
-          commentPosBuf += pos
+          val comment = Comment(pos, currentCommentBuf.toString)
+          currentCommentBuf.clear()
+          commentBuf += comment
 
           if (comment.isDocComment)
             addComment(comment)
@@ -1065,7 +1065,7 @@ object Scanners {
       else if (ch == '*') { nextChar(); skipComment(); finishComment() }
       else {
         // This was not a comment, remove the `/` from the buffer
-        commentBuf.clear()
+        currentCommentBuf.clear()
         false
       }
     }

compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala

@@ -83,8 +83,8 @@ object SyntaxHighlighting {
         }
       }
 
-      for (span <- scanner.commentSpans)
-        highlightPosition(span, CommentColor)
+      for (comment <- scanner.comments)
+        highlightPosition(comment.span, CommentColor)
 
       object TreeHighlighter extends untpd.UntypedTreeTraverser {
         import untpd._

compiler/src/dotty/tools/dotc/util/DiffUtil.scala

@@ -70,7 +70,9 @@ object DiffUtil {
    *         differences are highlighted.
    */
   def mkColoredLineDiff(expected: Seq[String], actual: Seq[String]): String = {
-    val expectedSize = EOF.length max expected.maxBy(_.length).length
+    val longestExpected = expected.map(_.length).maxOption.getOrElse(0)
+    val longestActual = actual.map(_.length).maxOption.getOrElse(0)
+    val expectedSize = EOF.length max longestActual max longestExpected
     actual.padTo(expected.length, "").zip(expected.padTo(actual.length, "")).map { case (act, exp) =>
       mkColoredLineDiff(exp, act, expectedSize)
     }.mkString(System.lineSeparator)
@@ -101,11 +103,75 @@ object DiffUtil {
         case Deleted(str) => deleted(str)
       }.mkString
 
+    (expectedDiff, actualDiff)
     val pad = " " * 0.max(expectedSize - expected.length)
 
     expectedDiff + pad + "  |  " + actualDiff
   }
 
+  private def ensureLineSeparator(str: String): String =
+    if str.endsWith(System.lineSeparator) then
+      str
+    else
+      str + System.lineSeparator
+
+  /**
+   * Returns a colored diffs by comparison of lines instead of tokens.
+   * It will automatically group subsequential pairs of `Insert` and `Delete`
+   * in order to improve the readability
+   *
+   * @param expected The expected lines
+   * @param actual   The actual lines
+   * @return A string with colored diffs between `expected` and `actual` grouped whenever possible
+   */
+  def mkColoredHorizontalLineDiff(expected: String, actual: String): String = {
+    val indent = 2
+    val tab = " " * indent
+    val insertIndent = "+" ++ (" " * (indent - 1))
+    val deleteIndent = "-" ++ (" " * (indent - 1))
+
+    if actual.isEmpty then
+      (expected.linesIterator.map(line => added(insertIndent + line)).toList :+ deleted("--- EMPTY OUTPUT ---"))
+        .map(ensureLineSeparator).mkString
+    else if expected.isEmpty then
+      (added("--- NO VALUE EXPECTED ---") +: actual.linesIterator.map(line => deleted(deleteIndent + line)).toList)
+        .map(ensureLineSeparator).mkString
+    else
+      lazy val diff = {
+        val expectedTokens = expected.linesWithSeparators.toArray
+        val actualTokens = actual.linesWithSeparators.toArray
+        hirschberg(actualTokens, expectedTokens)
+      }.toList
+
+      val transformedDiff = diff.flatMap {
+        case Modified(original, str) => Seq(
+          Inserted(ensureLineSeparator(original)), Deleted(ensureLineSeparator(str))
+        )
+        case other => Seq(other)
+      }
+
+      val zipped = transformedDiff zip transformedDiff.drop(1)
+
+      val (acc, inserts, deletions) = zipped.foldLeft((Seq[Patch](), Seq[Inserted](), Seq[Deleted]())): (acc, patches) =>
+        val (currAcc, inserts, deletions) = acc
+        patches match
+          case (currentPatch: Inserted, nextPatch: Deleted) =>
+            (currAcc, inserts :+ currentPatch, deletions)
+          case (currentPatch: Deleted, nextPatch: Inserted) =>
+            (currAcc, inserts, deletions :+ currentPatch)
+          case (currentPatch, nextPatch) =>
+            (currAcc :++ inserts :++ deletions :+ currentPatch, Seq.empty, Seq.empty)
+
+      val stackedDiff = acc :++ inserts :++ deletions :+ diff.last
+
+      stackedDiff.collect {
+        case Unmodified(str) => tab + str
+        case Inserted(str) => added(insertIndent + str)
+        case Deleted(str) => deleted(deleteIndent + str)
+      }.map(ensureLineSeparator).mkString
+
+  }
+
   def mkColoredCodeDiff(code: String, lastCode: String, printDiffDel: Boolean): String = {
     val tokens = splitTokens(code, Nil).toArray
     val lastTokens = splitTokens(lastCode, Nil).toArray

compiler/src/dotty/tools/dotc/util/SourceFile.scala

@@ -16,8 +16,9 @@ import scala.collection.mutable.ArrayBuffer
 import scala.util.chaining.given
 
 import java.io.File.separator
+import java.net.URI
 import java.nio.charset.StandardCharsets
-import java.nio.file.{FileSystemException, NoSuchFileException}
+import java.nio.file.{FileSystemException, NoSuchFileException, Paths}
 import java.util.Optional
 import java.util.concurrent.atomic.AtomicInteger
 import java.util.regex.Pattern
@@ -222,6 +223,13 @@ object SourceFile {
     SourceFile(new VirtualFile(name.replace(separator, "/"), content.getBytes(StandardCharsets.UTF_8)), content.toCharArray)
       .tap(_._maybeInComplete = maybeIncomplete)
 
+  /** A helper method to create a virtual source file for given URI.
+   *  It relies on SourceFile#virtual implementation to create the virtual file.
+   */
+  def virtual(uri: URI, content: String): SourceFile =
+    val path = Paths.get(uri).toString
+    SourceFile.virtual(path, content)
+
   /** Returns the relative path of `source` within the `reference` path
    *
    *  It returns the absolute path of `source` if it is not contained in `reference`.