Skip to content

Commit

Permalink
ci: replace npm with homegrown (#477)
Browse files Browse the repository at this point in the history 
there has been some bitrot in the CI tool, since no current version of npm can install all the packages it needs. it is also intensely slow and prone to crashes. it's also hard to make it always choose the newest packages.

- delay parsing some bits of package json until later. also make it more flexible

this replacement always pull the latest.
  • Loading branch information
@oyvindberg
oyvindberg authored Sep 20, 2022
1 parent 6976cf5 commit 1bc8f2a
Show file tree
Hide file tree
Showing 4 changed files with 294 additions and 97 deletions.
40 changes: 25 additions & 15 deletions importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/Source.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,36 +62,46 @@ object Source {
implicit val SourceFormatter: Formatter[Source] = _.libName.value

/* for files referenced through here we must shorten the paths */
def findShortenedFiles(src: Source.TsLibSource): IArray[InFile] = {
def fromTypingsJson(fromFolder: Source.FromFolder, fileOpt: Option[String]): IArray[InFile] =
fileOpt match {
case Some(path) if path.endsWith("typings.json") =>
def findShortenedFiles(src: Source): IArray[InFile] = {
def fromTypingsJson(fromFolder: Source.FromFolder, files: Option[IArray[String]]): IArray[InFile] =
files.getOrElse(IArray.Empty).collect {
case path if path.endsWith("typings.json") =>
val typingsJsonPath = fromFolder.folder.path / os.RelPath(path)
val typingsJson = Json.force[TypingsJson](typingsJsonPath)
IArray(InFile(typingsJsonPath / os.up / typingsJson.main))
case _ => Empty
InFile(typingsJsonPath / os.up / typingsJson.main)
}

def fromFileEntry(fromFolder: Source.FromFolder, fileOpt: Option[String]): IArray[InFile] =
IArray.fromOption(fileOpt.flatMap(file => LibraryResolver.file(fromFolder.folder, file)))
def fromFileEntry(fromFolder: Source.FromFolder, files: Option[IArray[String]]): IArray[InFile] =
files.getOrElse(IArray.Empty).mapNotNone(file => LibraryResolver.file(fromFolder.folder, file))

def fromModuleDeclaration(fromFolder: Source.FromFolder, fileOpt: Option[String]): IArray[InFile] =
fileOpt.flatMap(file => LibraryResolver.file(fromFolder.folder, file)) match {
case Some(existingFile) if Source.hasTypescriptSources(existingFile.folder) => IArray(existingFile)
case _ => Empty
def fromModuleDeclaration(
fromFolder: Source.FromFolder,
files: Option[Map[String, String]],
): IArray[InFile] = {
val files1 = files match {
case Some(files) => IArray.fromTraversable(files.values)
case None => IArray.Empty
}

files1
.mapNotNone(file => LibraryResolver.file(fromFolder.folder, file))
.mapNotNone {
case existingFile if Source.hasTypescriptSources(existingFile.folder) => Some(existingFile)
case _ => None
}
}

src match {
case _: StdLibSource => Empty
case f: FromFolder =>
val fromTypings =
IArray(
fromFileEntry(f, f.packageJsonOpt.flatMap(_.types).orElse(f.packageJsonOpt.flatMap(_.typings))),
fromTypingsJson(f, f.packageJsonOpt.flatMap(_.typings)),
fromFileEntry(f, f.packageJsonOpt.flatMap(_.parsedTypes).orElse(f.packageJsonOpt.flatMap(_.parsedTypings))),
fromTypingsJson(f, f.packageJsonOpt.flatMap(_.parsedTypings)),
).flatten

if (fromTypings.nonEmpty) fromTypings
else fromModuleDeclaration(f, f.packageJsonOpt.flatMap(_.module))
else fromModuleDeclaration(f, f.packageJsonOpt.flatMap(_.parsedModules))
}
}
}
7 changes: 5 additions & 2 deletions importer/src/main/scala/org/scalablytyped/converter/internal/importer/Ci.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,12 +215,15 @@ class Ci(config: Ci.Config, paths: Ci.Paths, pool: ForkJoinPool, ec: ExecutionCo
val external: NotNeededPackages =
Json.force[NotNeededPackages](dtFolder.path / os.up / "notNeededPackages.json")

val wanted = external.packages
.map(_._2.libraryName)
.toSet + TsIdentLibrary("typescript") ++ Libraries.extraExternals

UpToDateExternals(
interfaceLogger,
interfaceCmd,
files.existing(paths.cacheFolder / 'npm),
external.packages.map(_._2.libraryName).toSet + TsIdentLibrary("typescript") ++ Libraries.extraExternals,
config.conversion.ignoredLibs,
wanted -- config.conversion.ignoredLibs,
config.conserveSpace,
config.offline,
)
Expand Down
259 changes: 196 additions & 63 deletions ...rter/src/main/scala/org/scalablytyped/converter/internal/importer/UpToDateExternals.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,88 +2,122 @@ package org.scalablytyped.converter.internal
package importer

import com.olvind.logging.Logger
import gigahorse.support.okhttp.Gigahorse
import gigahorse.{HttpClient, Request}
import io.circe013.Decoder
import io.circe013.generic.semiauto.deriveDecoder
import io.circe013.syntax.EncoderOps
import org.scalablytyped.converter.internal.ts.{PackageJson, TsIdentLibrary}

import java.util.concurrent.Executors
import scala.collection.mutable
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.util.{Failure, Success, Try}

object UpToDateExternals {
class NpmClient(client: HttpClient) {
def getVersions(lib: TsIdentLibrary)(implicit ec: ExecutionContext): Future[Either[Throwable, NpmPackageVersions]] =
client
.run(Request(s"https://registry.npmjs.org/${lib.value}"))
.map(res => res.bodyAsString)
.transformWith {
case Failure(exception) => Future.successful(Left(exception))
case Success(string) =>
Json[NpmPackageVersions](string) match {
case Left(err) => Future.successful(Left(new RuntimeException(s"couldn't parse body: $string", err)))
case Right(value) => Future.successful(Right(value))
}
}

def download(dist: PackageJson.Dist, to: os.Path)(
implicit ec: ExecutionContext,
): Future[Either[Throwable, os.Path]] =
client
.download(Request(dist.tarball), to.toIO)
.map(res => os.Path(res))
.transformWith(tried => Future.successful(tried.toEither))
}

object NpmClient {
def apply(config: gigahorse.Config): NpmClient =
new NpmClient(Gigahorse.http(config))
}

case class NpmPackageVersions(
name: String,
`dist-tags`: Map[String, String],
versions: Map[String, PackageJson],
) {
def latestVersion: Option[String] =
`dist-tags`.get("latest").orElse {
versions.toList.sortBy { case (v, _) => v }.reverse.headOption.map { case (v, _) => v }
}
}

object NpmPackageVersions {
implicit val decodes: Decoder[NpmPackageVersions] = deriveDecoder
}

def apply(
logger: Logger[_],
cmd: Cmd,
folder: os.Path,
ensurePresentPackages: Set[TsIdentLibrary],
ignored: Set[TsIdentLibrary],
conserveSpace: Boolean,
offline: Boolean,
): InFolder = {

val packageJsonPath = folder / "package.json"
val nodeModulesPath = folder / "node_modules"
val packageJson = Json.opt[PackageJson](packageJsonPath)

val alreadyAddedExternals: Set[TsIdentLibrary] =
packageJson match {
case Some(packageJson) =>
packageJson.allLibs(dev = false, peer = true).keySet
case None =>
files.softWrite(packageJsonPath)(_.println("{}"))
Set.empty
}

val missingExternals: Set[TsIdentLibrary] =
ensurePresentPackages -- alreadyAddedExternals -- ignored
val packageJsonPath = folder / "package.json"
val nodeModulesPath = folder / "node_modules"
val existingPackageJson = Json.opt[PackageJson](packageJsonPath)

/* graalvm bundles a botched version which fails with SOE */
val npmCommand = sys.env.get("NVM_BIN") match {
case None => List("npm")
case Some(path) => List(s"$path/node", "--stack-size=4096", s"$path/npm")
}

if (missingExternals.isEmpty) logger.warn(s"All external libraries present in $nodeModulesPath")
else {
logger.warn(s"Adding ${missingExternals.size} missing libraries to $nodeModulesPath")
missingExternals.toSeq.map(_.value).sorted.grouped(100).foreach { es =>
cmd.runVerbose(
npmCommand,
"add",
"--ignore-scripts",
"--no-cache",
"--no-audit",
"--no-bin-links",
"--legacy-peer-deps",
es,
)(folder)
// if offline mode we only install what's missing. dont care that other libs may be outdated
val targets =
if (offline) {
val alreadyAdded = existingPackageJson.flatMap(_.dependencies).getOrElse(Map.empty)
val missingExternals = ensurePresentPackages -- alreadyAdded.keySet
if (missingExternals.isEmpty) logger.warn(s"All external libraries present in $nodeModulesPath")
else logger.warn(s"Adding ${missingExternals.size} missing libraries to $nodeModulesPath")
missingExternals
} else {
logger.warn(s"Updating libraries in $nodeModulesPath")
ensurePresentPackages
}
}

if (!offline) {
logger.warn(s"Updating libraries in $nodeModulesPath")
val npmClient = NpmClient(
Gigahorse.config.withMaxRequestRetry(3),
)
implicit val ec: ExecutionContext = ExecutionContext.fromExecutorService(Executors.newCachedThreadPool())

/* because of course *cough* somebody distributes their whole history, and `npm` refuses to update when that happens */
os.walk.stream(folder).foreach {
case dir if os.isDir(dir) && dir.last === ".git" => files.deleteAll(dir)
case _ => ()
val isStarted = mutable.Set[TsIdentLibrary]()
def shouldSkip(lib: TsIdentLibrary) =
synchronized {
if (isStarted.contains(lib)) true
else {
isStarted.add(lib)
false
}
}

cmd.runVerbose(
npmCommand,
'upgrade,
"--latest",
"--no-cache",
"--ignore-scripts",
"--no-audit",
"--no-bin-links",
)(folder)
}
targets.grouped(100).foldLeft(existingPackageJson.getOrElse(PackageJson.Empty)) {
case (packageJson, targets) =>
val alreadyAdded = packageJson.dependencies.getOrElse(Map.empty)
val results = Await.result(
downloadAll(cmd, nodeModulesPath, logger, npmClient, targets, alreadyAdded, shouldSkip),
Duration.Inf,
)
val newInstalled = results.collect { case x: Completed => x.lib -> x.version }.toMap
val newDependencies = alreadyAdded ++ newInstalled
val newPackageJson = packageJson.copy(dependencies = Some(newDependencies))

files.writeBytes(packageJsonPath.toNIO, newPackageJson.asJson.spaces2.getBytes)
logger
.withContext("numLibraries", newDependencies.size)
.withContext("newLibraries", newInstalled.size)
.warn("Flushing package.json to disk.")
newPackageJson

if (missingExternals.exists(_.value.startsWith("@material-ui")) || !offline) {
cmd.runVerbose(
npmCommand,
"add",
"@material-ui/core@3.9.3",
"--no-cache",
"--ignore-scripts",
"--no-audit",
"--no-bin-links",
)(folder)
}

if (conserveSpace && files.exists(folder)) {
Expand All @@ -102,4 +136,103 @@ object UpToDateExternals {

InFolder(nodeModulesPath)
}

sealed trait Result
case class Completed(lib: TsIdentLibrary, version: String) extends Result
case class Failed(lib: TsIdentLibrary) extends Result

def downloadAll(
cmd: Cmd,
nodeModules: os.Path,
logger: Logger[_],
npmClient: NpmClient,
libs: Set[TsIdentLibrary],
installed: Map[TsIdentLibrary, String],
shouldSkip: TsIdentLibrary => Boolean,
)(implicit ec: ExecutionContext): Future[List[Result]] = {
def ensureDownloaded(path: List[TsIdentLibrary], lib: TsIdentLibrary): Future[List[Result]] = {
val skip = shouldSkip(lib)

val libLogger = logger
.withContext("lib", lib.value)
.withContext("via", path.map(_.value).reverse.mkString(" -> "))

if (skip) Future.successful(Nil)
else {
npmClient.getVersions(lib).flatMap {
case Left(th) =>
libLogger.warn("Couldn't download", th)
Future.successful(List(Failed(lib)))

case Right(versions) =>
libLogger.info(s"fetched metadata")
versions.latestVersion match {
case Some(latestVersion) if installed.get(lib).contains(latestVersion) =>
Future.successful(List(Completed(lib, latestVersion)))

case Some(latestVersion) =>
versions.versions.get(latestVersion) match {
case Some(latestPackageJson) =>
val versionLogger = libLogger.withContext(latestVersion)
val download: Future[List[Result]] =
latestPackageJson.dist match {
case Some(dist) =>
versionLogger.info(s"downloading")
val tempDir = os.temp.dir()

npmClient.download(dist, (tempDir / "package.tar.gz")).map {
case Left(th) =>
versionLogger.warn("Couldn't download", th)
List(Failed(lib))
case Right(tempTarball) =>
Try {
val destination = nodeModules / os.RelPath(lib.value)
cmd.run("tar", "zxvf", tempTarball)(tempDir)
if (os.exists(destination)) os.remove.all(destination)
os.makeDir.all(destination / os.up)
// `package` subdirectory is from the downloaded tarball
val extractedDir = os.list(tempDir).filter(os.isDir).head
os.move(from = extractedDir, to = destination)
os.remove.all(tempDir)
} match {
case Failure(th) =>
versionLogger.warn("Couldn't write/extract", th)
List(Failed(lib))
case Success(_) =>
versionLogger.warn("downloaded")
List(Completed(lib, latestVersion))
}
}
case None =>
Future {
versionLogger.warn(s"Couldn't determine latest dist")
List(Failed(lib))
}
}

val downloadDependencies: List[Future[List[Result]]] =
latestPackageJson
.allLibs(dev = false, peer = true)
.map { case (depLib, _) => ensureDownloaded(lib :: path, depLib) }
.toList

Future.sequence(List(download) ++ downloadDependencies).map(_.flatten)

case None =>
Future {
libLogger.warn(s"Couldn't determine latest version")
List(Failed(lib))
}
}

case None =>
libLogger.warn("Couldn't determine latest version")
Future.successful(List(Failed(lib)))
}
}
}
}

Future.sequence(libs.map(lib => ensureDownloaded(Nil, lib))).map(_.toList.flatten)
}
}
Loading

0 comments on commit 1bc8f2a

Please sign in to comment.