drop circleci for github actions, run CI on mac and windows as well

.gitattributes

@@ -0,0 +1,2 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto

.github/workflows/ci-build.yml

@@ -0,0 +1,87 @@
+name: CI Build
+on:
+  pull_request:
+    branches: [ 'master' ]
+
+jobs:
+  scalafmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: coursier/setup-action@v1.2.0-M3
+        with:
+          apps: scalafmt
+
+      - uses: coursier/cache-action@v6
+
+      - name: Scalafmt
+        run: scalafmt -c .scalafmt.conf --check
+
+  build:
+    name: CI Build on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+          - os: macos-latest
+          - os: windows-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: coursier/cache-action@v6
+
+      - uses: coursier/setup-action@v1.2.0-M3
+        with:
+          jvm: adopt:11
+          apps: sbt
+
+      - name: Get tests folder sha
+        id: get-tests-folder-sha
+        run: |
+          echo "::set-output name=sha::$(git ls-tree HEAD tests --object-only)"
+
+      - uses: actions/cache@v3
+        with:
+          path: test-cache
+          key: ${{ runner.os }}-${{ steps.get-tests-folder-sha.outputs.sha }}-test-cache-v1
+
+      - name: Build and test
+        run: sbt test
+        env:
+          CI: true
+          CI_TEST_CACHE: ../test-cache
+
+# commented out while I figure out how to speed it up or when to run it
+#  sbt-scripted-tests:
+#    name: Run sbt scripted tests
+#    runs-on: ubuntu-latest
+#    timeout-minutes: 25
+#    steps:
+#      - uses: actions/checkout@v3
+#        with:
+#          fetch-depth: 0
+#
+#      - uses: coursier/cache-action@v6
+#
+#      # cache artifacts within same date. should help a bit with the compile time
+#      - name: Get timestamp
+#        id: get-date
+#        run: |
+#          echo "::set-output name=time::$(date +%yy-%m-%d)"
+#
+#      - uses: actions/cache@v3
+#        with:
+#          path: ~/.ivy2/local/org.scalablytyped
+#          key: scripted-v1-${{ steps.get-date.outputs.time }}
+#
+#      - uses: coursier/setup-action@v1.2.0-M3
+#        with:
+#          jvm: adopt:11
+#          apps: sbt
+#
+#      - name: Run scripted
+#        run: sbt scripted

.github/workflows/release-snapshot.yml

@@ -0,0 +1,18 @@
+# note that the proper release is made manually with the release script because it's
+# a heavy operation (builds all demo repos)
+# that's why we don't trigger on tag
+name: Release snapshot
+on:
+  push:
+    branches: [ 'master' ]
+jobs:
+  release-snapshot:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: coursier/cache-action@v6
+      - uses: coursier/setup-action@v1.2.0-M3
+        apps: sbt
+
+      - name: publish snapshot
+        run: sbt ci-release

core/src/main/scala/org/scalablytyped/converter/internal/Digest.scala

@@ -22,13 +22,11 @@ object Digest {
       (bs: Array[Byte]) => bs
   }
 
-  def of[T: Digestable](ts: scala.collection.Iterable[T]): Digest =
-    new Digest(
-      ts.foldLeft(MessageDigest.getInstance("MD5")) {
-          case (digest, t) =>
-            digest.update(Digestable[T].bytesFrom(t))
-            digest
-        }
-        .digest(),
-    )
+  def of[T <: AnyRef: Digestable](ts: IArray[T]): Digest = {
+    val md5 = MessageDigest.getInstance("MD5")
+    ts.foreach { t =>
+      md5.update(Digestable[T].bytesFrom(t))
+    }
+    new Digest(md5.digest())
+  }
 }

core/src/main/scala/org/scalablytyped/converter/internal/files.scala

@@ -54,29 +54,33 @@ object files {
   }
 
   def syncAbs(
-      absolutePathFiles: Map[os.Path, Array[Byte]],
+      absolutePathFiles: IArray[(os.Path, String)],
       folder:            os.Path,
       deleteUnknowns:    Boolean,
       soft:              Boolean,
   ): Unit = {
 
-    if (deleteUnknowns)
+    if (deleteUnknowns) {
+      val known = absolutePathFiles.map { case (p, _) => p }.toSet
       folder match {
         case f if files.exists(f) =>
           os.walk(f, IgnoreProjectFiles).foreach {
-            case p if os.isFile(p) && !absolutePathFiles.contains(p) => deleteAll(p)
-            case _                                                   => ()
+            case p if os.isFile(p) && !known(p) => deleteAll(p)
+            case _                              => ()
           }
         case _ => ()
       }
+    }
 
     absolutePathFiles.foreach {
       case (file, content) =>
-        if (soft) softWriteBytes(file.toNIO, content) else writeBytes(file.toNIO, content)
+        val bytes = content.getBytes(constants.Utf8)
+        if (soft) softWriteBytes(file.toNIO, bytes) else writeBytes(file.toNIO, bytes)
+        ()
     }
   }
 
-  def sync(fs: Map[os.RelPath, Array[Byte]], folder: os.Path, deleteUnknowns: Boolean, soft: Boolean): Unit =
+  def sync(fs: IArray[(os.RelPath, String)], folder: os.Path, deleteUnknowns: Boolean, soft: Boolean): Unit =
     syncAbs(fs.map { case (relPath, content) => folder / relPath -> content }, folder, deleteUnknowns, soft)
 
   def softWrite[T](path: os.Path)(f: PrintWriter => T): Synced =

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/ImportTree.scala

@@ -309,10 +309,6 @@ class ImportTree(
           }
 
         val approximationCandidates = findCandidates(tpe, depth = 0, inferred = Empty)
-        if (codePath.forceHasPath.codePath.parts.last.value == "PipelineDestination") {
-          val foo = findCandidates(tpe, depth = 0, inferred = Empty)
-          println(foo)
-        }
 
         val chosen: Option[TypeRef] =
           approximationCandidates

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/PersistingParser.scala

@@ -20,7 +20,7 @@ object PersistingParser {
               val shortestRelative =
                 inputFolders.map(f => file.path.relativeTo(f.path)).sortBy(_.toString.length).head.toString
               val base = cacheDir.resolve(s"${BuildInfo.version}").resolve(shortestRelative)
-              base.resolve(Digest.of(List(bs)).hexString)
+              base.resolve(Digest.of(IArray(bs)).hexString)
           },
           logger,
         ) {

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/Phase3Compile.scala

@@ -1,8 +1,6 @@
 package org.scalablytyped.converter.internal
 package importer
 
-import java.time.ZonedDateTime
-
 import com.olvind.logging.{Formatter, Logger}
 import org.scalablytyped.converter.internal.importer.build._
 import org.scalablytyped.converter.internal.importer.documentation.Npmjs
@@ -11,6 +9,7 @@ import org.scalablytyped.converter.internal.phases.{GetDeps, IsCircular, Phase,
 import org.scalablytyped.converter.internal.scalajs._
 import org.scalablytyped.converter.internal.scalajs.flavours.FlavourImpl
 
+import java.time.ZonedDateTime
 import scala.collection.immutable.SortedSet
 import scala.concurrent.Await
 import scala.concurrent.duration._
@@ -32,7 +31,7 @@ class Phase3Compile(
     ensureSourceFilesWritten:   Boolean,
 ) extends Phase[LibTsSource, LibScalaJs, PublishedSbtProject] {
 
-  val ScalaFiles: PartialFunction[(os.RelPath, Array[Byte]), Array[Byte]] = {
+  val ScalaFiles: PartialFunction[(os.RelPath, String), String] = {
     case (path, value)
         if path.ext === "scala" ||
           path.ext === "sbt" ||
@@ -66,23 +65,24 @@ class Phase3Compile(
         val resourcesDir  = os.RelPath("src") / 'main / 'resources
         val metadataOpt   = Try(Await.result(metadataFetcher(lib.source, logger), 2.seconds)).toOption.flatten
         val compilerPaths = CompilerPaths.of(versions, targetFolder, lib.libName)
-        val resources: Map[os.RelPath, Array[Byte]] =
+        val resources: IArray[(os.RelPath, String)] =
           if (generateScalaJsBundlerFile) ScalaJsBundlerDepFile(lib.source.libName, lib.libVersion)
-          else Map()
-
-        val sbtLayout = ContentSbtProject(
-          versions        = versions,
-          comments        = lib.packageTree.comments,
-          organization    = organization,
-          name            = lib.libName,
-          version         = VersionHack.TemplateValue,
-          localDeps       = deps.toIArrayValues,
-          deps            = flavour.dependencies,
-          scalaFiles      = scalaFiles.map { case (relPath, content) => sourcesDir / relPath -> content },
-          resources       = resources.map { case (relPath, content) => resourcesDir / relPath -> content },
-          metadataOpt     = metadataOpt,
-          declaredVersion = Some(lib.libVersion),
-        )
+          else Empty
+
+        val sbtLayout: SbtProjectLayout[os.RelPath, String] =
+          ContentSbtProject(
+            versions        = versions,
+            comments        = lib.packageTree.comments,
+            organization    = organization,
+            name            = lib.libName,
+            version         = VersionHack.TemplateValue,
+            localDeps       = deps.toIArrayValues,
+            deps            = flavour.dependencies,
+            scalaFiles      = scalaFiles.map { case (relPath, content) => sourcesDir / relPath -> content },
+            resources       = resources.map { case (relPath, content) => resourcesDir / relPath -> content },
+            metadataOpt     = metadataOpt,
+            declaredVersion = Some(lib.libVersion),
+          )
 
         val digest                = Digest.of(sbtLayout.all.collect(ScalaFiles))
         val finalVersion          = lib.libVersion.version(digest)
@@ -103,15 +103,20 @@ class Phase3Compile(
             files.sync(allFilesProperVersion.all, compilerPaths.baseDir, deleteUnknowns = true, soft = softWrites)
           }
 
-          if (existing.all.values.forall(files.exists)) {
+          if (existing.all.forall { case (_, file) => files.exists(file) }) {
             logger.warn(s"Using cached build $jarFile")
             PhaseRes.Ok(PublishedSbtProject(sbtProject)(compilerPaths.classesDir, existing, None))
           } else {
 
             files.deleteAll(compilerPaths.classesDir)
             os.makeDir.all(compilerPaths.classesDir)
             if (!ensureSourceFilesWritten) {
-              files.sync(allFilesProperVersion.all, compilerPaths.baseDir, deleteUnknowns = true, soft = softWrites)
+              files.sync(
+                allFilesProperVersion.all,
+                compilerPaths.baseDir,
+                deleteUnknowns = true,
+                soft           = softWrites,
+              )
             }
 
             val jarDeps: Set[Compiler.InternalDep] =

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/ScalaJsBundlerDepFile.scala

@@ -24,19 +24,19 @@ object ScalaJsBundlerDepFile {
     implicit val Decoder: Decoder[NpmDependencies] = io.circe013.generic.semiauto.deriveDecoder
   }
 
-  def apply(libName: TsIdentLibrary, v: LibraryVersion): Map[os.RelPath, Array[Byte]] =
+  def apply(libName: TsIdentLibrary, v: LibraryVersion): IArray[(os.RelPath, String)] =
     (v.libraryVersion, v.inGit) match {
       case (Some(version), None) if libName =/= ts.TsIdent.std =>
         val deps = List(Map(libName.value -> version))
-        Map(
+        IArray(
           os.RelPath(ScalaJsBundlerDepFile.manifestFileName) ->
             NpmDependencies(
               `compile-dependencies`    = deps,
               `test-dependencies`       = deps,
               `compile-devDependencies` = Nil,
               `test-devDependencies`    = Nil,
-            ).asJson.spaces2.getBytes(constants.Utf8),
+            ).asJson.spaces2,
         )
-      case _ => Map.empty
+      case _ => Empty
     }
 }

importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/build/ContentSbtProject.scala

@@ -15,11 +15,11 @@ object ContentSbtProject {
       version:         String,
       localDeps:       IArray[PublishedSbtProject],
       deps:            Set[Dep],
-      scalaFiles:      Map[os.RelPath, Array[Byte]],
-      resources:       Map[os.RelPath, Array[Byte]],
+      scalaFiles:      IArray[(os.RelPath, String)],
+      resources:       IArray[(os.RelPath, String)],
       metadataOpt:     Option[Npmjs.Data],
       declaredVersion: Option[LibraryVersion],
-  ): SbtProjectLayout[os.RelPath, Array[Byte]] = {
+  ): SbtProjectLayout[os.RelPath, String] = {
 
     val buildSbt = {
       val allDeps: IArray[Dep] = IArray.fromTraversable(deps) ++ IArray(versions.runtime) ++ localDeps.map(d =>
@@ -43,13 +43,13 @@ object ContentSbtProject {
       .map(dep => s"addSbtPlugin(${dep.asSbt})")
       .mkString("", "\n", "\n")
 
-    val readme: (os.RelPath, Array[Byte]) =
-      os.RelPath("readme.md") -> ProjectReadme(name, declaredVersion, metadataOpt, comments).getBytes(constants.Utf8)
+    val readme: (os.RelPath, String) =
+      os.RelPath("readme.md") -> ProjectReadme(name, declaredVersion, metadataOpt, comments)
 
     SbtProjectLayout(
-      os.RelPath("build.sbt") -> buildSbt.getBytes(constants.Utf8),
-      os.RelPath("project") / "build.properties" -> s"sbt.version=${Versions.sbtVersion}".getBytes(constants.Utf8),
-      os.RelPath("project") / "plugins.sbt" -> pluginsSbt.getBytes(constants.Utf8),
+      os.RelPath("build.sbt") -> buildSbt,
+      os.RelPath("project") / "build.properties" -> s"sbt.version=${Versions.sbtVersion}",
+      os.RelPath("project") / "plugins.sbt" -> pluginsSbt,
       readme,
       scalaFiles,
       resources,