feat: node support in codegen server (#762)

_tasks/dnt.ts

@@ -11,6 +11,7 @@ await Promise.all([
     package: {
       name: "capi",
       version: Deno.args[0]!,
+      type: "module",
       description: "Capi is a framework for crafting interactions with Substrate chains",
       license: "Apache-2.0",
       repository: "github:paritytech/capi",
@@ -63,6 +64,14 @@ await Promise.all([
         name: "@substrate/smoldot-light",
         version: "0.7.6",
       },
+      "https://deno.land/x/smoldot@light-js-deno-v0.7.6/client.d.ts": {
+        name: "@substrate/smoldot-light",
+        version: "0.7.6",
+      },
+      "http://localhost:4646/frame/wss/rpc.polkadot.io/@latest/mod.js": {
+        name: "@capi/polkadot",
+        version: "http://localhost:4646/frame/wss/rpc.polkadot.io/@v0.9.39/pkg.tar",
+      },
       // "https://raw.githubusercontent.com/paritytech/capi-crypto-wrappers/14289c5/lib.ts":
       //   "https://raw.githubusercontent.com/paritytech/capi-crypto-wrappers/14289c5/lib.node.ts",
     },
@@ -73,6 +82,7 @@ await Promise.all([
         test: true,
       },
       webSocket: true,
+      crypto: true,
     },
     test: false,
     typeCheck: false,

codegen/FrameCodegen.ts

@@ -35,10 +35,10 @@ export const chain = C.Rune.rec({ metadata, connection }).into(C.ChainRune)
     files.set(
       "chain.d.ts",
       `
-import * as _codecs from "./codecs.d.ts"
-import { connection } from "./connection.d.ts"
-import * as C from "./capi.d.ts"
-import * as t from "./types/mod.d.ts"
+import * as _codecs from "./codecs.js"
+import { connection } from "./connection.js"
+import * as C from "./capi.js"
+import * as t from "./types/mod.js"
 
 export const metadata: ${this.typeCodegen.print(this.metadata)}
 
@@ -55,19 +55,19 @@ export const chain: C.ChainRune<${this.chainName}, never>
       files.set(
         `mod.${ext}`,
         `
-export * from "./connection.${ext}"
-export * from "./chain.${ext}"
-export * from "./pallets.${ext}"
-export * as types from "./types/mod.${ext}"
+export * from "./connection.js"
+export * from "./chain.js"
+export * from "./pallets.js"
+export * as types from "./types/mod.js"
 `,
       )
 
       files.set(
         `pallets.${ext}`,
         `
-import { chain, ${isTypes ? this.chainName : ""} } from "./chain.${ext}"
-import * as C from "./capi.${ext}"
-import * as t from "./types/mod.${ext}"
+import { chain, ${isTypes ? this.chainName : ""} } from "./chain.js"
+import * as C from "./capi.js"
+import * as t from "./types/mod.js"
 
 ${
           Object.values(this.metadata.pallets).map((pallet) => {

codegen/TypeCodegen.ts

@@ -205,15 +205,15 @@ is${variant.tag}(value) {
       files.set(
         `${["types", ...path].join("/")}${isFolder ? "/mod" : ""}.${ext}`,
         `
-import * as C from "${rootDir}capi.${ext}"
-import * as _codecs from "${rootDir}codecs.${ext}"
-import * as t from "${rootDir}types/mod.${ext}"
+import * as C from "${rootDir}capi.js"
+import * as _codecs from "${rootDir}codecs.js"
+import * as t from "${rootDir}types/mod.js"
 
 ${
           [...groups].map(([name, entries]) => {
             const isFolder = this._write(files, [...path, name], entries)
             return `
-export * as ${name} from ${JSON.stringify(`./${name}${isFolder ? "/mod" : ""}.${ext}`)}`
+export * as ${name} from ${JSON.stringify(`./${name}${isFolder ? "/mod" : ""}.js`)}`
           }).join("\n")
         }
 
@@ -237,8 +237,8 @@ ${this.declVisitor.visit(codec)(name, isTypes)}
     files.set(
       "codecs.d.ts",
       `
-import * as C from "./capi.d.ts"
-import * as t from "./types/mod.d.ts"
+import * as C from "./capi.js"
+import * as t from "./types/mod.js"
 
 ${
         [...this.codecCodegen.codecIds.entries()].filter((x) => x[1] != null).map(([codec, id]) => `

deps/capi_crypto_wrappers.ts

@@ -1 +1 @@
-export * from "https://raw.githubusercontent.com/paritytech/capi-crypto-wrappers/14289c5/mod.ts"
+export * from "https://raw.githubusercontent.com/paritytech/capi-crypto-wrappers/5d55e35/mod.ts"

deps/smoldot.ts

@@ -1 +1,5 @@
+export type {
+  Client,
+  ClientOptions,
+} from "https://deno.land/x/smoldot@light-js-deno-v0.7.6/client.d.ts"
 export * from "https://deno.land/x/smoldot@light-js-deno-v0.7.6/index-deno.js"

deps/std/archive.ts

@@ -0,0 +1 @@
+export * from "https://deno.land/std@0.180.0/archive/mod.ts"

main.ts

@@ -53,6 +53,7 @@ const running = await fetch(`${href}capi_cwd`)
 
 if (!running) {
   await serve(handler(env), {
+    hostname: "[::]",
     port: +portStr,
     signal,
     onError(error) {

providers/frame/FrameProvider.ts

@@ -9,6 +9,7 @@ import { WeakMemo } from "../../util/mod.ts"
 import { normalizeIdent } from "../../util/normalize.ts"
 import { tsFormatter } from "../../util/tsFormatter.ts"
 import { withSignal } from "../../util/withSignal.ts"
+import { generateTar } from "./common.ts"
 
 export abstract class FrameProvider extends Provider {
   codegenCtxsPending: Record<string, Promise<FrameCodegen>> = {}
@@ -40,6 +41,15 @@ export abstract class FrameProvider extends Provider {
         async () => `export * from ${JSON.stringify(capiPath)}`,
       )
     }
+    if (filePath === "pkg.tar") {
+      return new Response(
+        await this.env.cache.getRaw(`${this.cacheKey(pathInfo)}/pkg.tar`, async () => {
+          const files = await this.codegen(pathInfo)
+          const chainName = await this.chainName(pathInfo)
+          return await generateTar(files, chainName, vRuntime)
+        }),
+      )
+    }
     return await f.code(this.env.cache, request, async () => {
       const codegen = await this.codegen(pathInfo)
       const code = codegen.get(filePath)

providers/frame/FrameProxyProvider.ts

@@ -64,11 +64,11 @@ export abstract class FrameProxyProvider extends FrameProvider {
   async connectionCode(pathInfo: PathInfo, isTypes: boolean) {
     const url = this.staticUrl(pathInfo)
     return `
-      import * as C from "./capi.${isTypes ? "d.ts" : "js"}"
+import * as C from "./capi.js"
 
-      export const discoveryValue ${isTypes ? ":" : "="} "${url}"
+export const discoveryValue ${isTypes ? ":" : "="} "${url}"
 
-      export const connection ${
+export const connection ${
       isTypes
         ? ": C.ConnectionRune<never>"
         : "= C.connection((signal) => C.WsConnection.connect(discoveryValue, signal))"

providers/frame/common.ts

@@ -1,5 +1,9 @@
 import { ss58 } from "../../crypto/mod.ts"
 import * as $ from "../../deps/scale.ts"
+import { Tar } from "../../deps/std/archive.ts"
+import { Buffer } from "../../deps/std/io.ts"
+import { readableStreamFromReader, writableStreamFromWriter } from "../../deps/std/streams.ts"
+import { tsFormatter } from "../../util/tsFormatter.ts"
 
 export const DEFAULT_TEST_USER_COUNT = 100_000
 const DEFAULT_TEST_USER_INITIAL_FUNDS = 1_000_000_000_000_000_000
@@ -67,3 +71,47 @@ export async function handleCount(request: Request, cache: { count: number }): P
     headers: { "Content-Type": "application/json" },
   })
 }
+
+export async function generateTar(_files: Map<string, string>, chainName: string, version: string) {
+  const files = new Map(_files)
+
+  files.set("capi.js", `export * from "capi"`)
+  files.set("capi.d.ts", `export * from "capi"`)
+  files.set(
+    "package.json",
+    JSON.stringify(
+      {
+        name: packageName(chainName),
+        version,
+        type: "module",
+        main: "./mod.js",
+        peerDependencies: {
+          capi: "*",
+        },
+      },
+      null,
+      2,
+    ),
+  )
+
+  const tar = new Tar()
+  for (const [name, content] of files) {
+    const formatted = /\.(js|ts)$/.test(name) ? tsFormatter.formatText(name, content) : content
+    const data = new TextEncoder().encode(formatted)
+    tar.append(`package/${name}`, {
+      contentSize: data.length,
+      reader: new Buffer(data),
+    })
+  }
+
+  const buffer = new Buffer()
+
+  await readableStreamFromReader(tar.getReader())
+    .pipeTo(writableStreamFromWriter(buffer))
+
+  return buffer.bytes()
+}
+
+function packageName(chainName: string) {
+  return `@capi/` + chainName.replace(/([a-z])(?=[A-Z])/g, (x) => `${x}-`).toLowerCase()
+}

rpc/smoldot.test.ts

@@ -65,7 +65,7 @@ Deno.test({
         await assertRejects(
           async () => {
             const connection = new SmoldotConnection({ relayChainSpec: "" })
-            return connection.smoldotChainPending
+            return connection["smoldotChainPending"]
           },
           AddChainError,
         )

rpc/smoldot.ts

@@ -1,5 +1,4 @@
-import { start } from "../deps/smoldot.ts"
-import { Client, ClientOptions } from "../deps/smoldot/client.d.ts"
+import { Client, ClientOptions, start } from "../deps/smoldot.ts"
 import { deferred } from "../deps/std/async.ts"
 import { Connection } from "./Connection.ts"
 import { RpcEgressMessage } from "./rpc_messages.ts"
@@ -14,7 +13,8 @@ export interface SmoldotRpcConnProps {
 let client: undefined | Client
 
 export class SmoldotConnection extends Connection {
-  smoldotChainPending
+  // private so that the types don't need to be referenced when packaged
+  private smoldotChainPending
   listening
   stopListening
 

rune/ValueRune.ts

@@ -15,12 +15,9 @@ type Access<T, K extends keyof T, _K = NonIndexSignatureKeys<T>> = [
   >,
 ][0]
 
-type GetPath<T, P> = P extends [infer K, ...infer Q] ? GetPath<
-    // @ts-ignore: we assume it's a valid key
-    T[K],
-    Q
-  >
-  : T
+/** @ts-ignore: assume it's a valid key */
+type GetPath<T, P> = P extends [infer K, ...infer Q] ? GetPath<T[K], Q> : T
+
 type EnsurePath<T, P> = never extends P ? P extends [infer K, ...infer Q] ? 
       | [K & keyof T, ...EnsurePath<T[K & keyof T], Q>]
       | [keyof T, ...PropertyKey[]]

util/notifier.ts

@@ -2,9 +2,11 @@ import { Deferred, deferred } from "../deps/std/async.ts"
 
 export class Notifier implements PromiseLike<void> {
   private _next: Deferred<void> = null!
-  // dprint-ignore (doesn't respect instantiation expression)
-  then<T1 = void, T2 = never>(...args: Parameters<typeof Notifier.prototype._next.then<T1, T2>>) {
-    return (this._next ??= deferred()).then(...args)
+  then<T1 = void, T2 = never>(
+    onfulfilled?: ((value: void) => T1 | PromiseLike<T1>) | null | undefined,
+    onrejected?: ((reason: any) => T2 | PromiseLike<T2>) | null | undefined,
+  ) {
+    return (this._next ??= deferred()).then(onfulfilled, onrejected)
   }
   emit() {
     this._next?.resolve()