MarkdownPart: move to separate mdoc-parser module

build.sbt

@@ -171,6 +171,12 @@ val excludePprint = ExclusionRule(organization = "com.lihaoyi")
 val excludeCollection =
   ExclusionRule(organization = "org.scala-lang.modules", name = "scala-collection-compat_2.13")
 
+lazy val parser = project
+  .settings(
+    sharedSettings,
+    moduleName := "mdoc-parser"
+  )
+
 lazy val cli = project
   .settings(
     sharedSettings,
@@ -197,6 +203,7 @@ lazy val cli = project
       )
     )
   )
+  .dependsOn(parser)
 
 lazy val mdoc = project
   .settings(
@@ -238,7 +245,7 @@ lazy val mdoc = project
       "com.lihaoyi" %% "pprint" % V.pprint
     )
   )
-  .dependsOn(runtime, cli)
+  .dependsOn(parser, runtime, cli)
   .enablePlugins(BuildInfoPlugin)
 
 lazy val testsInput = project
@@ -334,7 +341,7 @@ lazy val unit = project
       "testsInputClassDirectory" -> (testsInput / Compile / classDirectory).value
     )
   )
-  .dependsOn(mdoc, testsInput, tests)
+  .dependsOn(parser, mdoc, testsInput, tests)
   .enablePlugins(BuildInfoPlugin, MdocPlugin)
 
 lazy val unitJS = project
@@ -485,7 +492,7 @@ def localCrossPublish(versions: List[String]): Def.Initialize[Task[Unit]] =
     .reduceLeft(_ dependsOn _)
 
 def localCrossPublishProjects(scalaV: String): Def.Initialize[Task[Unit]] = {
-  val projects = List(runtime, cli, mdoc, js, jsWorker).reverse
+  val projects = List(parser, runtime, cli, mdoc, js, jsWorker).reverse
   projects
     .map(p => localCrossPublishProject(p, scalaV))
     .reduceLeft(_ dependsOn _)
@@ -494,7 +501,7 @@ def localCrossPublishProjects(scalaV: String): Def.Initialize[Task[Unit]] = {
 def localCrossPublishProject(ref: Project, scalaV: String): Def.Initialize[Task[Unit]] =
   Def.task {
     val versionValue = (ThisBuild / version).value
-    val projects = List(runtime, cli, mdoc, js, jsWorker)
+    val projects = List(parser, runtime, cli, mdoc, js, jsWorker)
     val setttings =
       (ThisBuild / version := versionValue) ::
         projects.map(p => p / scalaVersion := scalaV)

cli/src/main/scala/mdoc/internal/cli/Settings.scala

@@ -156,7 +156,7 @@ case class Settings(
     @Hidden()
     @Description("The Coursier logger used to report progress bars when downloading dependencies")
     coursierLogger: coursierapi.Logger = coursierapi.Logger.progressBars()
-) {
+) extends mdoc.parser.ParserSettings {
 
   val isMarkdownFileExtension = markdownExtensions.toSet
 

mdoc/src/main/scala/mdoc/internal/markdown/FenceInput.scala

@@ -8,6 +8,7 @@ import mdoc.internal.markdown.Modifier.Str
 import mdoc.internal.markdown.Modifier.Default
 import mdoc.internal.markdown.Modifier.Post
 import mdoc.internal.markdown.Modifier.Pre
+import mdoc.parser.{CodeFence, Text}
 
 case class PreFenceInput(block: CodeFence, input: Input, mod: Pre)
 case class StringFenceInput(block: CodeFence, input: Input, mod: Str)
@@ -55,8 +56,8 @@ class FenceInput(ctx: Context, baseInput: Input) {
 
   private def invalid(info: Text, message: String): Unit = {
     val offset = "scala mdoc:".length
-    val start = info.pos.start + offset
-    val end = info.pos.end - 1
+    val start = info.posBeg + offset
+    val end = info.posEnd - 1
     val pos = Position.Range(baseInput, start, end)
     ctx.reporter.error(pos, message)
   }
@@ -97,7 +98,7 @@ class FenceInput(ctx: Context, baseInput: Input) {
     getModifier(block.info) match {
       case Some(mod) =>
         if (isValid(block.info, mod)) {
-          val input = Input.Slice(baseInput, block.body.pos.start, block.body.pos.end)
+          val input = Input.Slice(baseInput, block.body.posBeg, block.body.posEnd)
           Some(ScalaFenceInput(block, input, mod))
         } else {
           None

mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala

@@ -129,7 +129,6 @@ object Markdown {
     val file = MarkdownFile.parse(
       textWithVariables,
       inputFile,
-      reporter,
       settings
     )
     val processor = new Processor()(context)

mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala

@@ -2,11 +2,12 @@ package mdoc.internal.markdown
 
 import scala.meta.inputs.Position
 import scala.meta.inputs.Input
-import mdoc.Reporter
 
 import scala.collection.mutable
 import scala.meta.io.RelativePath
-import mdoc.internal.cli.{InputFile, Settings}
+
+import mdoc.internal.cli.InputFile
+import mdoc.parser._
 
 final case class MarkdownFile(input: Input, file: InputFile, parts: List[MarkdownPart]) {
   private val appends = mutable.ListBuffer.empty[String]
@@ -21,161 +22,12 @@ final case class MarkdownFile(input: Input, file: InputFile, parts: List[Markdow
   }
 }
 object MarkdownFile {
-  object syntax {
-    private[markdown] implicit class StringOps(private val x: String) extends AnyVal {
-      def isNL: Boolean = x.forall { c => c == '\n' || c == '\r' }
-    }
-
-    private[markdown] implicit class StringBuilderOps(private val x: StringBuilder) extends AnyVal {
-      def appendLinesPrefixed(prefix: String, text: String): Unit = {
-        text.linesWithSeparators foreach { line =>
-          if (line.nonEmpty && !line.isNL && !line.startsWith(prefix)) x.append(prefix)
-          x.append(line)
-        }
-      }
-    }
-  }
-  sealed abstract class State
-  object State {
-    case class CodeFence(start: Int, backticks: String, info: String, indent: Int) extends State
-    case object Text extends State
-  }
-  class Parser(input: Input, reporter: Reporter, settings: Settings) {
-    private val text = input.text
-    private def newPos(start: Int, end: Int): Position = {
-      Position.Range(input, start, end)
-    }
-    private def newText(start: Int, end: Int): Text = {
-      val adaptedEnd = math.max(start, end)
-      val part = Text(text.substring(start, adaptedEnd))
-      part.pos = newPos(start, adaptedEnd)
-      part
-    }
-    private def newCodeFence(
-        state: State.CodeFence,
-        backtickStart: Int,
-        backtickEnd: Int
-    ): CodeFence = {
-      // tag is characters preceding the code fence in this line
-      val tag = newText(state.start, state.start + state.indent)
-      val open = newText(state.start, state.start + state.indent + state.backticks.length())
-        .dropLinePrefix(state.indent)
-      val info = newText(open.pos.end, open.pos.end + state.info.length())
-      val adaptedBacktickStart = math.max(0, backtickStart - 1)
-      val body = newText(info.pos.end, adaptedBacktickStart).dropLinePrefix(state.indent)
-      val close = newText(adaptedBacktickStart, backtickEnd).dropLinePrefix(state.indent)
-      val part = CodeFence(open, info, body, close, tag)
-      part.pos = newPos(state.start, backtickEnd)
-      part
-    }
-    def acceptParts(): List[MarkdownPart] = {
-      var state: State = State.Text
-      val parts = mutable.ListBuffer.empty[MarkdownPart]
-      var curr = 0
-      text.linesWithSeparators.foreach { line =>
-        val end = curr + line.length()
-        state match {
-          case State.Text =>
-            val start = line.indexOf("```")
-            if (start == 0 || (start > 0 && settings.allowCodeFenceIndented)) {
-              val fence = line.substring(start)
-              val backticks = fence.takeWhile(_ == '`')
-              val info = fence.substring(backticks.length())
-              state = State.CodeFence(curr, backticks, info, indent = start)
-            } else {
-              parts += newText(curr, end)
-            }
-          case s: State.CodeFence =>
-            val start = line.indexOf(s.backticks)
-            if (
-              start == s.indent &&
-              line.forall(ch => ch == '`' || ch.isWhitespace)
-            ) {
-              parts += newCodeFence(s, curr, end)
-              state = State.Text
-            }
-        }
-        curr += line.length()
-      }
-      state match {
-        case s: State.CodeFence =>
-          parts += newCodeFence(s, text.length(), text.length())
-        case _ =>
-      }
-      parts.toList
-    }
-  }
   def parse(
       input: Input,
       file: InputFile,
-      reporter: Reporter,
-      settings: Settings
+      settings: ParserSettings
   ): MarkdownFile = {
-    val parser = new Parser(input, reporter, settings)
-    val parts = parser.acceptParts()
+    val parts = MarkdownPart.parse(input.text, settings)
     MarkdownFile(input, file, parts)
   }
 }
-
-sealed abstract class MarkdownPart {
-  import MarkdownFile.syntax._
-
-  var pos: Position = Position.None
-  final def renderToString(out: StringBuilder): Unit =
-    this match {
-      case Text(value) =>
-        out.append(value)
-      case fence: CodeFence =>
-        val indentation = if (fence.hasBlankTag) fence.tag.value else " " * fence.indent
-        fence.newPart match {
-          case Some(newPart) =>
-            out.appendLinesPrefixed(indentation, newPart)
-          case None =>
-            fence.tag.renderToString(out)
-            fence.openBackticks.renderToString(out)
-            fence.newInfo match {
-              case None =>
-                fence.info.renderToString(out)
-              case Some(newInfo) =>
-                out.append(newInfo)
-                if (!newInfo.endsWith("\n")) {
-                  out.append("\n")
-                }
-            }
-            fence.newBody match {
-              case None =>
-                out.appendLinesPrefixed(indentation, fence.body.value)
-              case Some(newBody) =>
-                out.appendLinesPrefixed(indentation, newBody)
-            }
-            out.appendLinesPrefixed(indentation, fence.closeBackticks.value)
-        }
-    }
-}
-final case class Text(value: String) extends MarkdownPart {
-  import MarkdownFile.syntax._
-
-  def dropLinePrefix(indent: Int): Text = {
-    if (indent > 0) {
-      val updatedValue = value.linesWithSeparators.map { line =>
-        if (!line.isNL && line.length >= indent) line.substring(indent) else line
-      }.mkString
-      val updatedText = Text(updatedValue)
-      updatedText.pos = this.pos
-      updatedText
-    } else this
-  }
-}
-final case class CodeFence(
-    openBackticks: Text,
-    info: Text,
-    body: Text,
-    closeBackticks: Text,
-    tag: Text = Text("")
-) extends MarkdownPart {
-  var newPart = Option.empty[String]
-  var newInfo = Option.empty[String]
-  var newBody = Option.empty[String]
-  def indent: Int = tag.value.length
-  def hasBlankTag: Boolean = tag.value.forall(_.isWhitespace)
-}

mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala

@@ -28,6 +28,7 @@ import scala.meta.io.AbsolutePath
 import scala.meta.parsers.Parsed
 import scala.meta.Source
 import mdoc.internal.BuildInfo
+import mdoc.parser.CodeFence
 
 object MdocDialect {