From e53d1d4254e8b76183b94004283dd124a4e50890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Raddum=20Berg?= Date: Tue, 20 Sep 2022 23:32:03 +0200 Subject: [PATCH] ci: replace npm with homegrown there has been some bitrot in the CI tool, since no current version of npm can install all the packages it needs. it is also intensely slow and prone to crashes. it's also hard to make it always choose the newest packages. - delay parsing some bits of package json until later. also make it more flexible this replacement always pull the latest. --- .../converter/internal/importer/Source.scala | 40 ++- .../converter/internal/importer/Ci.scala | 7 +- .../internal/importer/UpToDateExternals.scala | 259 +++++++++++++----- .../converter/internal/ts/metadata.scala | 85 ++++-- 4 files changed, 294 insertions(+), 97 deletions(-) diff --git a/importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/Source.scala b/importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/Source.scala index e01776a6f8..fb20a2cf0c 100644 --- a/importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/Source.scala +++ b/importer-portable/src/main/scala/org/scalablytyped/converter/internal/importer/Source.scala @@ -62,36 +62,46 @@ object Source { implicit val SourceFormatter: Formatter[Source] = _.libName.value /* for files referenced through here we must shorten the paths */ - def findShortenedFiles(src: Source.TsLibSource): IArray[InFile] = { - def fromTypingsJson(fromFolder: Source.FromFolder, fileOpt: Option[String]): IArray[InFile] = - fileOpt match { - case Some(path) if path.endsWith("typings.json") => + def findShortenedFiles(src: Source): IArray[InFile] = { + def fromTypingsJson(fromFolder: Source.FromFolder, files: Option[IArray[String]]): IArray[InFile] = + files.getOrElse(IArray.Empty).collect { + case path if path.endsWith("typings.json") => val typingsJsonPath = fromFolder.folder.path / os.RelPath(path) val typingsJson = Json.force[TypingsJson](typingsJsonPath) - IArray(InFile(typingsJsonPath / os.up / typingsJson.main)) - case _ => Empty + InFile(typingsJsonPath / os.up / typingsJson.main) } - def fromFileEntry(fromFolder: Source.FromFolder, fileOpt: Option[String]): IArray[InFile] = - IArray.fromOption(fileOpt.flatMap(file => LibraryResolver.file(fromFolder.folder, file))) + def fromFileEntry(fromFolder: Source.FromFolder, files: Option[IArray[String]]): IArray[InFile] = + files.getOrElse(IArray.Empty).mapNotNone(file => LibraryResolver.file(fromFolder.folder, file)) - def fromModuleDeclaration(fromFolder: Source.FromFolder, fileOpt: Option[String]): IArray[InFile] = - fileOpt.flatMap(file => LibraryResolver.file(fromFolder.folder, file)) match { - case Some(existingFile) if Source.hasTypescriptSources(existingFile.folder) => IArray(existingFile) - case _ => Empty + def fromModuleDeclaration( + fromFolder: Source.FromFolder, + files: Option[Map[String, String]], + ): IArray[InFile] = { + val files1 = files match { + case Some(files) => IArray.fromTraversable(files.values) + case None => IArray.Empty } + files1 + .mapNotNone(file => LibraryResolver.file(fromFolder.folder, file)) + .mapNotNone { + case existingFile if Source.hasTypescriptSources(existingFile.folder) => Some(existingFile) + case _ => None + } + } + src match { case _: StdLibSource => Empty case f: FromFolder => val fromTypings = IArray( - fromFileEntry(f, f.packageJsonOpt.flatMap(_.types).orElse(f.packageJsonOpt.flatMap(_.typings))), - fromTypingsJson(f, f.packageJsonOpt.flatMap(_.typings)), + fromFileEntry(f, f.packageJsonOpt.flatMap(_.parsedTypes).orElse(f.packageJsonOpt.flatMap(_.parsedTypings))), + fromTypingsJson(f, f.packageJsonOpt.flatMap(_.parsedTypings)), ).flatten if (fromTypings.nonEmpty) fromTypings - else fromModuleDeclaration(f, f.packageJsonOpt.flatMap(_.module)) + else fromModuleDeclaration(f, f.packageJsonOpt.flatMap(_.parsedModules)) } } } diff --git a/importer/src/main/scala/org/scalablytyped/converter/internal/importer/Ci.scala b/importer/src/main/scala/org/scalablytyped/converter/internal/importer/Ci.scala index f240401e71..c7d35e3bb5 100644 --- a/importer/src/main/scala/org/scalablytyped/converter/internal/importer/Ci.scala +++ b/importer/src/main/scala/org/scalablytyped/converter/internal/importer/Ci.scala @@ -215,12 +215,15 @@ class Ci(config: Ci.Config, paths: Ci.Paths, pool: ForkJoinPool, ec: ExecutionCo val external: NotNeededPackages = Json.force[NotNeededPackages](dtFolder.path / os.up / "notNeededPackages.json") + val wanted = external.packages + .map(_._2.libraryName) + .toSet + TsIdentLibrary("typescript") ++ Libraries.extraExternals + UpToDateExternals( interfaceLogger, interfaceCmd, files.existing(paths.cacheFolder / 'npm), - external.packages.map(_._2.libraryName).toSet + TsIdentLibrary("typescript") ++ Libraries.extraExternals, - config.conversion.ignoredLibs, + wanted -- config.conversion.ignoredLibs, config.conserveSpace, config.offline, ) diff --git a/importer/src/main/scala/org/scalablytyped/converter/internal/importer/UpToDateExternals.scala b/importer/src/main/scala/org/scalablytyped/converter/internal/importer/UpToDateExternals.scala index a61416b3d2..50ce91c9c4 100644 --- a/importer/src/main/scala/org/scalablytyped/converter/internal/importer/UpToDateExternals.scala +++ b/importer/src/main/scala/org/scalablytyped/converter/internal/importer/UpToDateExternals.scala @@ -2,88 +2,122 @@ package org.scalablytyped.converter.internal package importer import com.olvind.logging.Logger +import gigahorse.support.okhttp.Gigahorse +import gigahorse.{HttpClient, Request} +import io.circe013.Decoder +import io.circe013.generic.semiauto.deriveDecoder +import io.circe013.syntax.EncoderOps import org.scalablytyped.converter.internal.ts.{PackageJson, TsIdentLibrary} +import java.util.concurrent.Executors +import scala.collection.mutable +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.util.{Failure, Success, Try} + object UpToDateExternals { + class NpmClient(client: HttpClient) { + def getVersions(lib: TsIdentLibrary)(implicit ec: ExecutionContext): Future[Either[Throwable, NpmPackageVersions]] = + client + .run(Request(s"https://registry.npmjs.org/${lib.value}")) + .map(res => res.bodyAsString) + .transformWith { + case Failure(exception) => Future.successful(Left(exception)) + case Success(string) => + Json[NpmPackageVersions](string) match { + case Left(err) => Future.successful(Left(new RuntimeException(s"couldn't parse body: $string", err))) + case Right(value) => Future.successful(Right(value)) + } + } + + def download(dist: PackageJson.Dist, to: os.Path)( + implicit ec: ExecutionContext, + ): Future[Either[Throwable, os.Path]] = + client + .download(Request(dist.tarball), to.toIO) + .map(res => os.Path(res)) + .transformWith(tried => Future.successful(tried.toEither)) + } + + object NpmClient { + def apply(config: gigahorse.Config): NpmClient = + new NpmClient(Gigahorse.http(config)) + } + + case class NpmPackageVersions( + name: String, + `dist-tags`: Map[String, String], + versions: Map[String, PackageJson], + ) { + def latestVersion: Option[String] = + `dist-tags`.get("latest").orElse { + versions.toList.sortBy { case (v, _) => v }.reverse.headOption.map { case (v, _) => v } + } + } + + object NpmPackageVersions { + implicit val decodes: Decoder[NpmPackageVersions] = deriveDecoder + } + def apply( logger: Logger[_], cmd: Cmd, folder: os.Path, ensurePresentPackages: Set[TsIdentLibrary], - ignored: Set[TsIdentLibrary], conserveSpace: Boolean, offline: Boolean, ): InFolder = { - val packageJsonPath = folder / "package.json" - val nodeModulesPath = folder / "node_modules" - val packageJson = Json.opt[PackageJson](packageJsonPath) - - val alreadyAddedExternals: Set[TsIdentLibrary] = - packageJson match { - case Some(packageJson) => - packageJson.allLibs(dev = false, peer = true).keySet - case None => - files.softWrite(packageJsonPath)(_.println("{}")) - Set.empty - } - - val missingExternals: Set[TsIdentLibrary] = - ensurePresentPackages -- alreadyAddedExternals -- ignored + val packageJsonPath = folder / "package.json" + val nodeModulesPath = folder / "node_modules" + val existingPackageJson = Json.opt[PackageJson](packageJsonPath) - /* graalvm bundles a botched version which fails with SOE */ - val npmCommand = sys.env.get("NVM_BIN") match { - case None => List("npm") - case Some(path) => List(s"$path/node", "--stack-size=4096", s"$path/npm") - } - - if (missingExternals.isEmpty) logger.warn(s"All external libraries present in $nodeModulesPath") - else { - logger.warn(s"Adding ${missingExternals.size} missing libraries to $nodeModulesPath") - missingExternals.toSeq.map(_.value).sorted.grouped(100).foreach { es => - cmd.runVerbose( - npmCommand, - "add", - "--ignore-scripts", - "--no-cache", - "--no-audit", - "--no-bin-links", - "--legacy-peer-deps", - es, - )(folder) + // if offline mode we only install what's missing. dont care that other libs may be outdated + val targets = + if (offline) { + val alreadyAdded = existingPackageJson.flatMap(_.dependencies).getOrElse(Map.empty) + val missingExternals = ensurePresentPackages -- alreadyAdded.keySet + if (missingExternals.isEmpty) logger.warn(s"All external libraries present in $nodeModulesPath") + else logger.warn(s"Adding ${missingExternals.size} missing libraries to $nodeModulesPath") + missingExternals + } else { + logger.warn(s"Updating libraries in $nodeModulesPath") + ensurePresentPackages } - } - if (!offline) { - logger.warn(s"Updating libraries in $nodeModulesPath") + val npmClient = NpmClient( + Gigahorse.config.withMaxRequestRetry(3), + ) + implicit val ec: ExecutionContext = ExecutionContext.fromExecutorService(Executors.newCachedThreadPool()) - /* because of course *cough* somebody distributes their whole history, and `npm` refuses to update when that happens */ - os.walk.stream(folder).foreach { - case dir if os.isDir(dir) && dir.last === ".git" => files.deleteAll(dir) - case _ => () + val isStarted = mutable.Set[TsIdentLibrary]() + def shouldSkip(lib: TsIdentLibrary) = + synchronized { + if (isStarted.contains(lib)) true + else { + isStarted.add(lib) + false + } } - cmd.runVerbose( - npmCommand, - 'upgrade, - "--latest", - "--no-cache", - "--ignore-scripts", - "--no-audit", - "--no-bin-links", - )(folder) - } + targets.grouped(100).foldLeft(existingPackageJson.getOrElse(PackageJson.Empty)) { + case (packageJson, targets) => + val alreadyAdded = packageJson.dependencies.getOrElse(Map.empty) + val results = Await.result( + downloadAll(cmd, nodeModulesPath, logger, npmClient, targets, alreadyAdded, shouldSkip), + Duration.Inf, + ) + val newInstalled = results.collect { case x: Completed => x.lib -> x.version }.toMap + val newDependencies = alreadyAdded ++ newInstalled + val newPackageJson = packageJson.copy(dependencies = Some(newDependencies)) + + files.writeBytes(packageJsonPath.toNIO, newPackageJson.asJson.spaces2.getBytes) + logger + .withContext("numLibraries", newDependencies.size) + .withContext("newLibraries", newInstalled.size) + .warn("Flushing package.json to disk.") + newPackageJson - if (missingExternals.exists(_.value.startsWith("@material-ui")) || !offline) { - cmd.runVerbose( - npmCommand, - "add", - "@material-ui/core@3.9.3", - "--no-cache", - "--ignore-scripts", - "--no-audit", - "--no-bin-links", - )(folder) } if (conserveSpace && files.exists(folder)) { @@ -102,4 +136,103 @@ object UpToDateExternals { InFolder(nodeModulesPath) } + + sealed trait Result + case class Completed(lib: TsIdentLibrary, version: String) extends Result + case class Failed(lib: TsIdentLibrary) extends Result + + def downloadAll( + cmd: Cmd, + nodeModules: os.Path, + logger: Logger[_], + npmClient: NpmClient, + libs: Set[TsIdentLibrary], + installed: Map[TsIdentLibrary, String], + shouldSkip: TsIdentLibrary => Boolean, + )(implicit ec: ExecutionContext): Future[List[Result]] = { + def ensureDownloaded(path: List[TsIdentLibrary], lib: TsIdentLibrary): Future[List[Result]] = { + val skip = shouldSkip(lib) + + val libLogger = logger + .withContext("lib", lib.value) + .withContext("via", path.map(_.value).reverse.mkString(" -> ")) + + if (skip) Future.successful(Nil) + else { + npmClient.getVersions(lib).flatMap { + case Left(th) => + libLogger.warn("Couldn't download", th) + Future.successful(List(Failed(lib))) + + case Right(versions) => + libLogger.info(s"fetched metadata") + versions.latestVersion match { + case Some(latestVersion) if installed.get(lib).contains(latestVersion) => + Future.successful(List(Completed(lib, latestVersion))) + + case Some(latestVersion) => + versions.versions.get(latestVersion) match { + case Some(latestPackageJson) => + val versionLogger = libLogger.withContext(latestVersion) + val download: Future[List[Result]] = + latestPackageJson.dist match { + case Some(dist) => + versionLogger.info(s"downloading") + val tempDir = os.temp.dir() + + npmClient.download(dist, (tempDir / "package.tar.gz")).map { + case Left(th) => + versionLogger.warn("Couldn't download", th) + List(Failed(lib)) + case Right(tempTarball) => + Try { + val destination = nodeModules / os.RelPath(lib.value) + cmd.run("tar", "zxvf", tempTarball)(tempDir) + if (os.exists(destination)) os.remove.all(destination) + os.makeDir.all(destination / os.up) + // `package` subdirectory is from the downloaded tarball + val extractedDir = os.list(tempDir).filter(os.isDir).head + os.move(from = extractedDir, to = destination) + os.remove.all(tempDir) + } match { + case Failure(th) => + versionLogger.warn("Couldn't write/extract", th) + List(Failed(lib)) + case Success(_) => + versionLogger.warn("downloaded") + List(Completed(lib, latestVersion)) + } + } + case None => + Future { + versionLogger.warn(s"Couldn't determine latest dist") + List(Failed(lib)) + } + } + + val downloadDependencies: List[Future[List[Result]]] = + latestPackageJson + .allLibs(dev = false, peer = true) + .map { case (depLib, _) => ensureDownloaded(lib :: path, depLib) } + .toList + + Future.sequence(List(download) ++ downloadDependencies).map(_.flatten) + + case None => + Future { + libLogger.warn(s"Couldn't determine latest version") + List(Failed(lib)) + } + } + + case None => + libLogger.warn("Couldn't determine latest version") + Future.successful(List(Failed(lib))) + } + } + } + } + + Future.sequence(libs.map(lib => ensureDownloaded(Nil, lib))).map(_.toList.flatten) + } } diff --git a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/metadata.scala b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/metadata.scala index 338016cf92..9c6d52344e 100644 --- a/ts/src/main/scala/org/scalablytyped/converter/internal/ts/metadata.scala +++ b/ts/src/main/scala/org/scalablytyped/converter/internal/ts/metadata.scala @@ -1,7 +1,8 @@ package org.scalablytyped.converter.internal package ts -import io.circe013.{Decoder, Encoder} +import io.circe013.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe013.{Decoder, Encoder, Json} import org.scalablytyped.converter.internal.maps._ import org.scalablytyped.converter.internal.orphanCodecs._ @@ -21,8 +22,8 @@ case class CompilerOptions( ) object CompilerOptions { - implicit val encodes: Encoder[CompilerOptions] = io.circe013.generic.semiauto.deriveEncoder - implicit val decodes: Decoder[CompilerOptions] = io.circe013.generic.semiauto.deriveDecoder + implicit val encodes: Encoder[CompilerOptions] = deriveEncoder + implicit val decodes: Decoder[CompilerOptions] = deriveDecoder } case class TsConfig( @@ -31,8 +32,8 @@ case class TsConfig( ) object TsConfig { - implicit val encodes: Encoder[TsConfig] = io.circe013.generic.semiauto.deriveEncoder - implicit val decodes: Decoder[TsConfig] = io.circe013.generic.semiauto.deriveDecoder + implicit val encodes: Encoder[TsConfig] = deriveEncoder + implicit val decodes: Decoder[TsConfig] = deriveDecoder } case class PackageJson( @@ -40,27 +41,77 @@ case class PackageJson( dependencies: Option[Map[TsIdentLibrary, String]], devDependencies: Option[Map[TsIdentLibrary, String]], peerDependencies: Option[Map[TsIdentLibrary, String]], - typings: Option[String], - module: Option[String], - types: Option[String], + typings: Option[Json], + module: Option[Json], + types: Option[Json], files: Option[IArray[String]], + dist: Option[PackageJson.Dist], ) { def allLibs(dev: Boolean, peer: Boolean): SortedMap[TsIdentLibrary, String] = smash(IArray.fromOptions(dependencies, devDependencies.filter(_ => dev), peerDependencies.filter(_ => peer))).toSorted + + def parsedTypes: Option[IArray[String]] = + types + .map { types => + types.fold( + IArray.Empty, + _ => sys.error(s"unexpected boolean in types structure: $types"), + _ => sys.error(s"unexpected number in types structure: $types"), + IArray(_), + arr => IArray.fromTraversable(arr).mapNotNone(_.asString), + _ => sys.error(s"unexpected object in types structure: $types"), + ) + } + .filter(_.nonEmpty) + + def parsedTypings: Option[IArray[String]] = + typings + .map { typings => + typings.fold( + IArray.Empty, + _ => sys.error(s"unexpected boolean in typings structure: $typings"), + _ => sys.error(s"unexpected number in typings structure: $typings"), + IArray(_), + arr => IArray.fromTraversable(arr).mapNotNone(_.asString), + _ => sys.error(s"unexpected object in typings structure: $typings"), + ) + } + .filter(_.nonEmpty) + + def parsedModules: Option[Map[String, String]] = { + def look(json: Json): Map[String, String] = + json.fold[Map[String, String]]( + Map.empty, + _ => Map.empty, + _ => Map.empty, + str => Map("" -> str), + _ => Map.empty, + obj => obj.toMap.flatMap { case (name, value) => value.asString.map(str => (name, str)) }, + ) + + module.map(look).filter(_.nonEmpty) + } } object PackageJson { - val Empty: PackageJson = PackageJson(None, None, None, None, None, None, None, None) + case class Dist(tarball: String) + + object Dist { + implicit val encodes: Encoder[Dist] = deriveEncoder + implicit val decodesDist: Decoder[Dist] = deriveDecoder + } + + val Empty: PackageJson = PackageJson(None, None, None, None, None, None, None, None, None) - implicit val encodes: Encoder[PackageJson] = io.circe013.generic.semiauto.deriveEncoder - implicit val decodes: Decoder[PackageJson] = io.circe013.generic.semiauto.deriveDecoder + implicit val encodes: Encoder[PackageJson] = deriveEncoder + implicit val decodes: Decoder[PackageJson] = deriveDecoder } case class NotNeededPackage(libraryName: TsIdentLibrary, asOfVersion: String) object NotNeededPackage { - implicit val encodes: Encoder[NotNeededPackage] = io.circe013.generic.semiauto.deriveEncoder - implicit val decodes: Decoder[NotNeededPackage] = io.circe013.generic.semiauto.deriveDecoder + implicit val encodes: Encoder[NotNeededPackage] = deriveEncoder + implicit val decodes: Decoder[NotNeededPackage] = deriveDecoder } case class TypingsJson( @@ -71,16 +122,16 @@ case class TypingsJson( ) object TypingsJson { - implicit val encodes: Encoder[TypingsJson] = io.circe013.generic.semiauto.deriveEncoder - implicit val decodes: Decoder[TypingsJson] = io.circe013.generic.semiauto.deriveDecoder + implicit val encodes: Encoder[TypingsJson] = deriveEncoder + implicit val decodes: Decoder[TypingsJson] = deriveDecoder } case class NotNeededPackages(packages: Map[String, NotNeededPackage]) object NotNeededPackages { - implicit val encodes: Encoder[NotNeededPackages] = io.circe013.generic.semiauto.deriveEncoder + implicit val encodes: Encoder[NotNeededPackages] = deriveEncoder implicit val decodes: Decoder[NotNeededPackages] = { - val old = io.circe013.generic.semiauto.deriveDecoder[NotNeededPackages] + val old = deriveDecoder[NotNeededPackages] // at some point they changed the format, this is the new one val array = Decoder[Vector[NotNeededPackage]]