From 11c7f36ae3ed98b5073642ce5cb5b0343bab6bfc Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 27 May 2024 18:32:38 -0400 Subject: [PATCH 001/122] User api routes --- packages/core/src/bin/utils/run.ts | 7 +- packages/core/src/build/service.ts | 71 +++++++++++++++- packages/core/src/server/service.ts | 123 ++++++++++++++-------------- 3 files changed, 135 insertions(+), 66 deletions(-) diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 29da6dc86..4bf2e624b 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -101,7 +101,12 @@ export async function run({ db: database.readonlyDb, }); - const server = await createServer({ common, graphqlSchema, readonlyStore }); + const server = await createServer({ + app: build.app, + common, + graphqlSchema, + readonlyStore, + }); // This can be a long-running operation, so it's best to do it after // starting the server so the app can become responsive more quickly. diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index a2f473990..35cc13747 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { readFileSync } from "node:fs"; +import fs from "node:fs"; import path from "node:path"; import type { Common } from "@/common/common.js"; import type { Config, OptionsConfig } from "@/config/config.js"; @@ -10,6 +10,7 @@ import type { Schema } from "@/schema/common.js"; import { buildGraphqlSchema } from "@/server/graphql/buildGraphqlSchema.js"; import { glob } from "glob"; import type { GraphQLSchema } from "graphql"; +import type { Hono } from "hono"; import { type ViteDevServer, createServer } from "vite"; import { ViteNodeRunner } from "vite-node/client"; import { ViteNodeServer } from "vite-node/server"; @@ -26,6 +27,8 @@ import { safeBuildSchema } from "./schema.js"; import { parseViteNodeError } from "./stacktrace.js"; const BUILD_ID_VERSION = "1"; +// TODO(kyle) use option for server +const SERVER_FILE = "_server.ts"; export type Service = { // static @@ -51,6 +54,8 @@ export type Build = { graphqlSchema: GraphQLSchema; // Indexing functions indexingFunctions: IndexingFunctions; + // Server + app?: Hono; }; export type BuildResult = @@ -64,6 +69,7 @@ type RawBuild = { indexingFunctions: RawIndexingFunctions; contentHash: string; }; + server: { app?: Hono }; }; export const create = async ({ @@ -153,11 +159,12 @@ export const start = async ( ): Promise => { const { common } = buildService; - const [configResult, schemaResult, indexingFunctionsResult] = + const [configResult, schemaResult, indexingFunctionsResult, serverResult] = await Promise.all([ executeConfig(buildService), executeSchema(buildService), executeIndexingFunctions(buildService), + executeServer(buildService), ]); if (configResult.status === "error") { @@ -169,11 +176,15 @@ export const start = async ( if (indexingFunctionsResult.status === "error") { return { status: "error", error: indexingFunctionsResult.error }; } + if (serverResult.status === "error") { + return { status: "error", error: serverResult.error }; + } const rawBuild: RawBuild = { config: configResult, schema: schemaResult, indexingFunctions: indexingFunctionsResult, + server: serverResult, }; const buildResult = await validateAndBuild(buildService, rawBuild); @@ -228,6 +239,9 @@ export const start = async ( const hasIndexingFunctionUpdate = invalidated.some((file) => buildService.srcRegex.test(file), ); + const hasServerUpdate = invalidated.includes( + SERVER_FILE.replace(/\\/g, "/"), + ); // This branch could trigger if you change a `note.txt` file within `src/`. // Note: We could probably do a better job filtering out files in `isFileIgnored`. @@ -269,6 +283,15 @@ export const start = async ( rawBuild.indexingFunctions = result; } + if (hasServerUpdate) { + const result = await executeServer(buildService); + if (result.status === "error") { + onBuild({ status: "error", error: result.error }); + return; + } + rawBuild.server = result; + } + const buildResult = await validateAndBuild(buildService, rawBuild); onBuild(buildResult); }; @@ -391,7 +414,7 @@ const executeIndexingFunctions = async ( const hash = createHash("sha256"); for (const file of files) { try { - const contents = readFileSync(file, "utf-8"); + const contents = fs.readFileSync(file, "utf-8"); hash.update(contents); } catch (e) { buildService.common.logger.warn({ @@ -406,6 +429,47 @@ const executeIndexingFunctions = async ( return { status: "success", indexingFunctions, contentHash }; }; +const executeServer = async ( + buildService: Service, +): Promise< + | { + status: "success"; + app?: Hono; + } + | { status: "error"; error: Error } +> => { + const doesServerExist = fs.existsSync( + path.join(buildService.common.options.srcDir, SERVER_FILE), + ); + + if (doesServerExist === false) { + return { status: "success" }; + } + + const executeResult = await executeFile(buildService, { + file: path.join(buildService.common.options.srcDir, SERVER_FILE), + }); + + if (executeResult.status === "error") { + buildService.common.logger.error({ + service: "build", + msg: `Error while executing '${path.relative( + buildService.common.options.rootDir, + SERVER_FILE, + )}':`, + error: executeResult.error, + }); + + return executeResult; + } + + const app = executeResult.exports.default as Hono; + + // TODO: check export default instanceof Hono + + return { status: "success", app }; +}; + const validateAndBuild = async ( { common }: Pick, rawBuild: RawBuild, @@ -476,6 +540,7 @@ const validateAndBuild = async ( graphqlSchema, indexingFunctions: buildConfigAndIndexingFunctionsResult.indexingFunctions, + app: rawBuild.server.app, }, }; }; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 4891a519c..884754a17 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -1,22 +1,14 @@ import http from "node:http"; import type { Common } from "@/common/common.js"; import type { ReadonlyStore } from "@/indexing-store/store.js"; -import { graphiQLHtml } from "@/ui/graphiql.html.js"; import { startClock } from "@/utils/timer.js"; -import { maxAliasesPlugin } from "@escape.tech/graphql-armor-max-aliases"; -import { maxDepthPlugin } from "@escape.tech/graphql-armor-max-depth"; -import { maxTokensPlugin } from "@escape.tech/graphql-armor-max-tokens"; import { serve } from "@hono/node-server"; -import { GraphQLError, GraphQLSchema } from "graphql"; -import { createYoga } from "graphql-yoga"; +import { GraphQLSchema } from "graphql"; import { Hono } from "hono"; import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; -import { - type GetLoader, - buildLoaderCache, -} from "./graphql/buildLoaderCache.js"; +import { type GetLoader } from "./graphql/buildLoaderCache.js"; type Server = { hono: Hono<{ Variables: { store: ReadonlyStore; getLoader: GetLoader } }>; @@ -26,15 +18,15 @@ type Server = { }; export async function createServer({ - graphqlSchema, - readonlyStore, + app, common, }: { + app?: Hono; graphqlSchema: GraphQLSchema; readonlyStore: ReadonlyStore; common: Common; }): Promise { - const hono = new Hono<{ + const ponderApp = new Hono<{ Variables: { store: ReadonlyStore; getLoader: GetLoader }; }>(); @@ -78,38 +70,38 @@ export async function createServer({ } }); - const createGraphqlYoga = (path: string) => - createYoga({ - schema: graphqlSchema, - context: () => { - const getLoader = buildLoaderCache({ store: readonlyStore }); - return { store: readonlyStore, getLoader }; - }, - graphqlEndpoint: path, - maskedErrors: process.env.NODE_ENV === "production", - logging: false, - graphiql: false, - parserAndValidationCache: false, - plugins: [ - maxTokensPlugin({ n: common.options.graphqlMaxOperationTokens }), - maxDepthPlugin({ - n: common.options.graphqlMaxOperationDepth, - ignoreIntrospection: false, - }), - maxAliasesPlugin({ - n: common.options.graphqlMaxOperationAliases, - allowList: [], - }), - ], - }); - - const rootYoga = createGraphqlYoga("/"); - const rootGraphiql = graphiQLHtml("/"); - - const prodYoga = createGraphqlYoga("/graphql"); - const prodGraphiql = graphiQLHtml("/graphql"); - - hono + // const createGraphqlYoga = (path: string) => + // createYoga({ + // schema: graphqlSchema, + // context: () => { + // const getLoader = buildLoaderCache({ store: readonlyStore }); + // return { store: readonlyStore, getLoader }; + // }, + // graphqlEndpoint: path, + // maskedErrors: process.env.NODE_ENV === "production", + // logging: false, + // graphiql: false, + // parserAndValidationCache: false, + // plugins: [ + // maxTokensPlugin({ n: common.options.graphqlMaxOperationTokens }), + // maxDepthPlugin({ + // n: common.options.graphqlMaxOperationDepth, + // ignoreIntrospection: false, + // }), + // maxAliasesPlugin({ + // n: common.options.graphqlMaxOperationAliases, + // allowList: [], + // }), + // ], + // }); + + // const rootYoga = createGraphqlYoga("/"); + // const rootGraphiql = graphiQLHtml("/"); + + // const prodYoga = createGraphqlYoga("/graphql"); + // const prodGraphiql = graphiQLHtml("/graphql"); + + ponderApp .use(cors()) .use(metricsMiddleware) .get("/metrics", async (c) => { @@ -138,23 +130,24 @@ export async function createServer({ return c.text("Historical indexing is not complete.", 503); }) - // Renders GraphiQL - .get("/graphql", (c) => c.html(prodGraphiql)) - // Serves GraphQL POST requests following healthcheck rules - .post("/graphql", (c) => { - if (isHealthy === false) { - return c.json( - { errors: [new GraphQLError("Historical indexing is not complete")] }, - 503, - ); - } - - return prodYoga.handle(c.req.raw); - }) - // Renders GraphiQL - .get("/", (c) => c.html(rootGraphiql)) - // Serves GraphQL POST requests regardless of health status, e.g. "dev UI" - .post("/", (c) => rootYoga.handle(c.req.raw)); + .get("/ready", async (c) => c.text("", 200)); + // Renders GraphiQL + // .get("/graphql", (c) => c.html(prodGraphiql)) + // // Serves GraphQL POST requests following healthcheck rules + // .post("/graphql", (c) => { + // if (isHealthy === false) { + // return c.json( + // { errors: [new GraphQLError("Historical indexing is not complete")] }, + // 503, + // ); + // } + + // return prodYoga.handle(c.req.raw); + // }) + // // Renders GraphiQL + // .get("/", (c) => c.html(rootGraphiql)) + // // Serves GraphQL POST requests regardless of health status, e.g. "dev UI" + // .post("/", (c) => rootYoga.handle(c.req.raw)); const createServerWithNextAvailablePort: typeof http.createServer = ( ...args: any @@ -190,6 +183,12 @@ export async function createServer({ return httpServer; }; + const hono = new Hono<{ + Variables: { store: ReadonlyStore; getLoader: GetLoader }; + }>() + .route("/_ponder", ponderApp) + .route("/", app); + const httpServer = await new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject(new Error("HTTP server failed to start within 5 seconds.")); From 8358ebd28e78e879669ccc23d1c4471760558f1c Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 28 May 2024 11:39:42 -0400 Subject: [PATCH 002/122] graphql server --- packages/core/src/build/plugin.ts | 35 ++++++++++++++++++++++++------ packages/core/src/build/service.ts | 4 ++-- packages/core/src/index.ts | 2 ++ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index cca7bf2f5..604e06e5e 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -1,10 +1,16 @@ +import path from "node:path"; +import type { Common } from "@/common/common.js"; import MagicString from "magic-string"; import type { Plugin } from "vite"; +import { SERVER_FILE } from "./service.js"; -export const regex = +export const ponderRegex = /^import\s+\{[^}]*\bponder\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; -export const shim = `export let ponder = { +export const serverRegex = + /^import\s+\{[^}]*\bGraphQLServer\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; + +export const ponderShim = `export let ponder = { fns: [], on(name, fn) { this.fns.push({ name, fn }); @@ -12,7 +18,15 @@ export const shim = `export let ponder = { }; `; -export function replaceStateless(code: string) { +export const serverShim = `import schema from "../ponder.schema.js"; + import { graphqlServer } from "@hono/graphql-server"; + import { buildGraphqlSchema } from "@ponder/core"; + async function GraphQLServer() { + return graphqlServer({ schema: buildGraphqlSchema(schema) }); + } +`; + +export function replaceStateless(code: string, regex: RegExp, shim: string) { const s = new MagicString(code); // MagicString.replace calls regex.exec(), which increments `lastIndex` // on a match. We have to set this back to zero to use the same regex @@ -22,15 +36,22 @@ export function replaceStateless(code: string) { return s; } -export const vitePluginPonder = (): Plugin => { +export const vitePluginPonder = (common: Common): Plugin => { return { name: "ponder", transform: (code, id) => { - if (regex.test(code)) { - const s = replaceStateless(code); + if ( + id === path.join(common.options.srcDir, SERVER_FILE) && + serverRegex.test(code) + ) { + const s = replaceStateless(code, serverRegex, serverShim); + const transformed = s.toString(); + const sourcemap = s.generateMap({ source: id }); + return { code: transformed, map: sourcemap }; + } else if (ponderRegex.test(code)) { + const s = replaceStateless(code, ponderRegex, ponderShim); const transformed = s.toString(); const sourcemap = s.generateMap({ source: id }); - return { code: transformed, map: sourcemap }; } else { return null; diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 35cc13747..378d3e3c6 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -28,7 +28,7 @@ import { parseViteNodeError } from "./stacktrace.js"; const BUILD_ID_VERSION = "1"; // TODO(kyle) use option for server -const SERVER_FILE = "_server.ts"; +export const SERVER_FILE = "_server.ts"; export type Service = { // static @@ -116,7 +116,7 @@ export const create = async ({ publicDir: false, customLogger: viteLogger, server: { hmr: false }, - plugins: [viteTsconfigPathsPlugin(), vitePluginPonder()], + plugins: [viteTsconfigPathsPlugin(), vitePluginPonder(common)], }); // This is Vite boilerplate (initializes the Rollup container). diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2b389fc94..6c1e1925b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -16,3 +16,5 @@ export type ContractConfig = Prettify; export type NetworkConfig = Prettify; export type BlockConfig = Prettify; export type DatabaseConfig = Prettify; + +export { buildGraphqlSchema } from "@/server/graphql/buildGraphqlSchema.js"; From 96de148d61213d68ec1d9e45745330bf52236ef7 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 28 May 2024 11:40:16 -0400 Subject: [PATCH 003/122] move graphql --- packages/core/src/bin/utils/run.test.ts | 2 +- packages/core/src/build/service.ts | 2 +- .../core/src/{server => }/graphql/buildGraphqlSchema.test.ts | 0 packages/core/src/{server => }/graphql/buildGraphqlSchema.ts | 0 packages/core/src/{server => }/graphql/buildLoaderCache.ts | 0 packages/core/src/{server => }/graphql/entity.ts | 0 packages/core/src/{server => }/graphql/enum.ts | 0 packages/core/src/{server => }/graphql/filter.test.ts | 0 packages/core/src/{server => }/graphql/filter.ts | 0 packages/core/src/{server => }/graphql/plural.ts | 0 packages/core/src/{server => }/graphql/scalar.ts | 0 packages/core/src/{server => }/graphql/singular.ts | 0 packages/core/src/index.ts | 2 +- packages/core/src/server/service.test.ts | 2 +- packages/core/src/server/service.ts | 2 +- 15 files changed, 5 insertions(+), 5 deletions(-) rename packages/core/src/{server => }/graphql/buildGraphqlSchema.test.ts (100%) rename packages/core/src/{server => }/graphql/buildGraphqlSchema.ts (100%) rename packages/core/src/{server => }/graphql/buildLoaderCache.ts (100%) rename packages/core/src/{server => }/graphql/entity.ts (100%) rename packages/core/src/{server => }/graphql/enum.ts (100%) rename packages/core/src/{server => }/graphql/filter.test.ts (100%) rename packages/core/src/{server => }/graphql/filter.ts (100%) rename packages/core/src/{server => }/graphql/plural.ts (100%) rename packages/core/src/{server => }/graphql/scalar.ts (100%) rename packages/core/src/{server => }/graphql/singular.ts (100%) diff --git a/packages/core/src/bin/utils/run.test.ts b/packages/core/src/bin/utils/run.test.ts index 4d428819f..79485344e 100644 --- a/packages/core/src/bin/utils/run.test.ts +++ b/packages/core/src/bin/utils/run.test.ts @@ -5,8 +5,8 @@ import { } from "@/_test/setup.js"; import type { Build } from "@/build/index.js"; import * as codegen from "@/common/codegen.js"; +import { buildGraphqlSchema } from "@/graphql/buildGraphqlSchema.js"; import { createSchema } from "@/schema/schema.js"; -import { buildGraphqlSchema } from "@/server/graphql/buildGraphqlSchema.js"; import { promiseWithResolvers } from "@ponder/common"; import { beforeEach, expect, test, vi } from "vitest"; import { run } from "./run.js"; diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 378d3e3c6..f0399e834 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -6,8 +6,8 @@ import type { Config, OptionsConfig } from "@/config/config.js"; import type { DatabaseConfig } from "@/config/database.js"; import type { Network } from "@/config/networks.js"; import type { EventSource } from "@/config/sources.js"; +import { buildGraphqlSchema } from "@/graphql/buildGraphqlSchema.js"; import type { Schema } from "@/schema/common.js"; -import { buildGraphqlSchema } from "@/server/graphql/buildGraphqlSchema.js"; import { glob } from "glob"; import type { GraphQLSchema } from "graphql"; import type { Hono } from "hono"; diff --git a/packages/core/src/server/graphql/buildGraphqlSchema.test.ts b/packages/core/src/graphql/buildGraphqlSchema.test.ts similarity index 100% rename from packages/core/src/server/graphql/buildGraphqlSchema.test.ts rename to packages/core/src/graphql/buildGraphqlSchema.test.ts diff --git a/packages/core/src/server/graphql/buildGraphqlSchema.ts b/packages/core/src/graphql/buildGraphqlSchema.ts similarity index 100% rename from packages/core/src/server/graphql/buildGraphqlSchema.ts rename to packages/core/src/graphql/buildGraphqlSchema.ts diff --git a/packages/core/src/server/graphql/buildLoaderCache.ts b/packages/core/src/graphql/buildLoaderCache.ts similarity index 100% rename from packages/core/src/server/graphql/buildLoaderCache.ts rename to packages/core/src/graphql/buildLoaderCache.ts diff --git a/packages/core/src/server/graphql/entity.ts b/packages/core/src/graphql/entity.ts similarity index 100% rename from packages/core/src/server/graphql/entity.ts rename to packages/core/src/graphql/entity.ts diff --git a/packages/core/src/server/graphql/enum.ts b/packages/core/src/graphql/enum.ts similarity index 100% rename from packages/core/src/server/graphql/enum.ts rename to packages/core/src/graphql/enum.ts diff --git a/packages/core/src/server/graphql/filter.test.ts b/packages/core/src/graphql/filter.test.ts similarity index 100% rename from packages/core/src/server/graphql/filter.test.ts rename to packages/core/src/graphql/filter.test.ts diff --git a/packages/core/src/server/graphql/filter.ts b/packages/core/src/graphql/filter.ts similarity index 100% rename from packages/core/src/server/graphql/filter.ts rename to packages/core/src/graphql/filter.ts diff --git a/packages/core/src/server/graphql/plural.ts b/packages/core/src/graphql/plural.ts similarity index 100% rename from packages/core/src/server/graphql/plural.ts rename to packages/core/src/graphql/plural.ts diff --git a/packages/core/src/server/graphql/scalar.ts b/packages/core/src/graphql/scalar.ts similarity index 100% rename from packages/core/src/server/graphql/scalar.ts rename to packages/core/src/graphql/scalar.ts diff --git a/packages/core/src/server/graphql/singular.ts b/packages/core/src/graphql/singular.ts similarity index 100% rename from packages/core/src/server/graphql/singular.ts rename to packages/core/src/graphql/singular.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 6c1e1925b..f038b02b9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -17,4 +17,4 @@ export type NetworkConfig = Prettify; export type BlockConfig = Prettify; export type DatabaseConfig = Prettify; -export { buildGraphqlSchema } from "@/server/graphql/buildGraphqlSchema.js"; +export { buildGraphqlSchema } from "@/graphql/buildGraphqlSchema.js"; diff --git a/packages/core/src/server/service.test.ts b/packages/core/src/server/service.test.ts index 7edde08ad..d153166b8 100644 --- a/packages/core/src/server/service.test.ts +++ b/packages/core/src/server/service.test.ts @@ -8,7 +8,7 @@ import { createSchema } from "@/schema/schema.js"; import { encodeCheckpoint, zeroCheckpoint } from "@/utils/checkpoint.js"; import type { GraphQLSchema } from "graphql"; import { beforeEach, expect, test, vi } from "vitest"; -import { buildGraphqlSchema } from "./graphql/buildGraphqlSchema.js"; +import { buildGraphqlSchema } from "../graphql/buildGraphqlSchema.js"; import { createServer } from "./service.js"; beforeEach(setupCommon); diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 884754a17..e92414379 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -8,7 +8,7 @@ import { Hono } from "hono"; import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; -import { type GetLoader } from "./graphql/buildLoaderCache.js"; +import { type GetLoader } from "../graphql/buildLoaderCache.js"; type Server = { hono: Hono<{ Variables: { store: ReadonlyStore; getLoader: GetLoader } }>; From dd7423894c9446eafe37e76a5abdb27856c23e20 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 28 May 2024 11:56:48 -0400 Subject: [PATCH 004/122] graphql middleware --- packages/core/package.json | 1 + packages/core/src/build/plugin.ts | 9 +++-- packages/core/src/graphql/middleware.ts | 11 ++++++ packages/core/src/index.ts | 2 +- packages/core/src/server/service.ts | 48 ------------------------- 5 files changed, 17 insertions(+), 54 deletions(-) create mode 100644 packages/core/src/graphql/middleware.ts diff --git a/packages/core/package.json b/packages/core/package.json index 16ebbe8c0..7a3f8b9fe 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -47,6 +47,7 @@ "@escape.tech/graphql-armor-max-aliases": "^2.3.0", "@escape.tech/graphql-armor-max-depth": "^2.2.0", "@escape.tech/graphql-armor-max-tokens": "^2.3.0", + "@hono/graphql-server": "^0.4.3", "@hono/node-server": "^1.8.2", "@ponder/utils": "workspace:*", "abitype": "^0.10.2", diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 604e06e5e..d85c30bf5 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -8,7 +8,7 @@ export const ponderRegex = /^import\s+\{[^}]*\bponder\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; export const serverRegex = - /^import\s+\{[^}]*\bGraphQLServer\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; + /^import\s+\{[^}]*\bGraphQLServer\b[^}]*\}\s+from\s+["']@ponder\/core["'];?.*$/gm; export const ponderShim = `export let ponder = { fns: [], @@ -19,10 +19,9 @@ export const ponderShim = `export let ponder = { `; export const serverShim = `import schema from "../ponder.schema.js"; - import { graphqlServer } from "@hono/graphql-server"; - import { buildGraphqlSchema } from "@ponder/core"; - async function GraphQLServer() { - return graphqlServer({ schema: buildGraphqlSchema(schema) }); + import { GraphQLServer as _GraphQLServer } from "@ponder/core"; + function GraphQLServer() { + return _GraphQLServer({ schema }); } `; diff --git a/packages/core/src/graphql/middleware.ts b/packages/core/src/graphql/middleware.ts new file mode 100644 index 000000000..90ef7b875 --- /dev/null +++ b/packages/core/src/graphql/middleware.ts @@ -0,0 +1,11 @@ +import type { Schema } from "@/schema/common.js"; +import { graphqlServer } from "@hono/graphql-server"; +import { buildGraphqlSchema } from "./buildGraphqlSchema.js"; + +// @ts-ignore +export function GraphQLServer(): ReturnType; +export function GraphQLServer({ + schema, +}: { schema: Schema }): ReturnType { + return graphqlServer({ schema: buildGraphqlSchema(schema) }); +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f038b02b9..fcdb1700f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -17,4 +17,4 @@ export type NetworkConfig = Prettify; export type BlockConfig = Prettify; export type DatabaseConfig = Prettify; -export { buildGraphqlSchema } from "@/graphql/buildGraphqlSchema.js"; +export { GraphQLServer } from "@/graphql/middleware.js"; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index e92414379..adb0591ec 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -70,37 +70,6 @@ export async function createServer({ } }); - // const createGraphqlYoga = (path: string) => - // createYoga({ - // schema: graphqlSchema, - // context: () => { - // const getLoader = buildLoaderCache({ store: readonlyStore }); - // return { store: readonlyStore, getLoader }; - // }, - // graphqlEndpoint: path, - // maskedErrors: process.env.NODE_ENV === "production", - // logging: false, - // graphiql: false, - // parserAndValidationCache: false, - // plugins: [ - // maxTokensPlugin({ n: common.options.graphqlMaxOperationTokens }), - // maxDepthPlugin({ - // n: common.options.graphqlMaxOperationDepth, - // ignoreIntrospection: false, - // }), - // maxAliasesPlugin({ - // n: common.options.graphqlMaxOperationAliases, - // allowList: [], - // }), - // ], - // }); - - // const rootYoga = createGraphqlYoga("/"); - // const rootGraphiql = graphiQLHtml("/"); - - // const prodYoga = createGraphqlYoga("/graphql"); - // const prodGraphiql = graphiQLHtml("/graphql"); - ponderApp .use(cors()) .use(metricsMiddleware) @@ -131,23 +100,6 @@ export async function createServer({ return c.text("Historical indexing is not complete.", 503); }) .get("/ready", async (c) => c.text("", 200)); - // Renders GraphiQL - // .get("/graphql", (c) => c.html(prodGraphiql)) - // // Serves GraphQL POST requests following healthcheck rules - // .post("/graphql", (c) => { - // if (isHealthy === false) { - // return c.json( - // { errors: [new GraphQLError("Historical indexing is not complete")] }, - // 503, - // ); - // } - - // return prodYoga.handle(c.req.raw); - // }) - // // Renders GraphiQL - // .get("/", (c) => c.html(rootGraphiql)) - // // Serves GraphQL POST requests regardless of health status, e.g. "dev UI" - // .post("/", (c) => rootYoga.handle(c.req.raw)); const createServerWithNextAvailablePort: typeof http.createServer = ( ...args: any From 41a81cf26a730c291afbe3039629dd1e4c487b44 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 28 May 2024 12:48:24 -0400 Subject: [PATCH 005/122] . --- packages/core/src/build/plugin.ts | 14 ++++++++------ packages/core/src/build/service.ts | 4 ++-- packages/core/src/common/codegen.ts | 3 +++ packages/core/src/graphql/middleware.ts | 8 +++----- packages/core/src/index.ts | 2 +- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index d85c30bf5..017dbbd08 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -8,7 +8,7 @@ export const ponderRegex = /^import\s+\{[^}]*\bponder\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; export const serverRegex = - /^import\s+\{[^}]*\bGraphQLServer\b[^}]*\}\s+from\s+["']@ponder\/core["'];?.*$/gm; + /^import\s+\{[^}]*\bhono\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; export const ponderShim = `export let ponder = { fns: [], @@ -18,11 +18,13 @@ export const ponderShim = `export let ponder = { }; `; -export const serverShim = `import schema from "../ponder.schema.js"; - import { GraphQLServer as _GraphQLServer } from "@ponder/core"; - function GraphQLServer() { - return _GraphQLServer({ schema }); - } +export const serverShim = `import { Hono } from "hono"; +import { createGraphQLMiddleware } from "@ponder/core"; +import schema from "../ponder.schema.js"; +export let hono = new Hono(); +function graphQLMiddleware() { + return createGraphQLMiddleware({ schema }); +} `; export function replaceStateless(code: string, regex: RegExp, shim: string) { diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index f0399e834..894c2e0d7 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -463,9 +463,9 @@ const executeServer = async ( return executeResult; } - const app = executeResult.exports.default as Hono; + const app = executeResult.exports.hono as Hono; - // TODO: check export default instanceof Hono + // TODO: check export instanceof Hono return { status: "success", app }; }; diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index 05fde059c..d6c9bc575 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -11,12 +11,15 @@ export const ponderEnv = `// This file enables type checking and editor autocomp declare module "@/generated" { import type { Virtual } from "@ponder/core"; + import type { Hono } from "hono"; type config = typeof import("./ponder.config.ts").default; type schema = typeof import("./ponder.schema.ts").default; export const ponder: Virtual.Registry; + export const hono: Hono; + export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< config, diff --git a/packages/core/src/graphql/middleware.ts b/packages/core/src/graphql/middleware.ts index 90ef7b875..a9648898d 100644 --- a/packages/core/src/graphql/middleware.ts +++ b/packages/core/src/graphql/middleware.ts @@ -2,10 +2,8 @@ import type { Schema } from "@/schema/common.js"; import { graphqlServer } from "@hono/graphql-server"; import { buildGraphqlSchema } from "./buildGraphqlSchema.js"; -// @ts-ignore -export function GraphQLServer(): ReturnType; -export function GraphQLServer({ +export const createGraphQLMiddleware = ({ schema, -}: { schema: Schema }): ReturnType { +}: { schema: Schema }): ReturnType => { return graphqlServer({ schema: buildGraphqlSchema(schema) }); -} +}; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index fcdb1700f..c2956faec 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -17,4 +17,4 @@ export type NetworkConfig = Prettify; export type BlockConfig = Prettify; export type DatabaseConfig = Prettify; -export { GraphQLServer } from "@/graphql/middleware.js"; +export { createGraphQLMiddleware } from "@/graphql/middleware.js"; From 8827016455ea6c36f3a8f33984d7cf310e2674e8 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 11 Jun 2024 22:32:03 -0400 Subject: [PATCH 006/122] example server --- examples/reference-erc20/package.json | 1 + examples/reference-erc20/ponder-env.d.ts | 3 + examples/reference-erc20/src/_server.ts | 7 +++ package.json | 1 - packages/core/src/utils/timer.ts | 3 +- pnpm-lock.yaml | 72 ++++++++++++++---------- 6 files changed, 55 insertions(+), 32 deletions(-) create mode 100644 examples/reference-erc20/src/_server.ts diff --git a/examples/reference-erc20/package.json b/examples/reference-erc20/package.json index 2398e90c5..d0bc6a886 100644 --- a/examples/reference-erc20/package.json +++ b/examples/reference-erc20/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.1.3", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index 1d5dc7ebe..7c4952bd8 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -5,12 +5,15 @@ declare module "@/generated" { import type { Virtual } from "@ponder/core"; + import type { Hono } from "hono"; type config = typeof import("./ponder.config.ts").default; type schema = typeof import("./ponder.schema.ts").default; export const ponder: Virtual.Registry; + export const hono: Hono; + export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< config, diff --git a/examples/reference-erc20/src/_server.ts b/examples/reference-erc20/src/_server.ts new file mode 100644 index 000000000..e285c9096 --- /dev/null +++ b/examples/reference-erc20/src/_server.ts @@ -0,0 +1,7 @@ +import { hono } from "@/generated"; + +hono.get("/kevin", (c) => { + return c.text("kevin"); +}); + +export { hono }; diff --git a/package.json b/package.json index 2891ce336..61d100486 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,6 @@ "packageManager": "pnpm@8.6.10", "pnpm": { "patchedDependencies": { - "graphql@16.8.1": "patches/graphql@16.8.1.patch", "detect-package-manager@3.0.1": "patches/detect-package-manager@3.0.1.patch" }, "peerDependencyRules": { diff --git a/packages/core/src/utils/timer.ts b/packages/core/src/utils/timer.ts index 7325750dc..cc87708e5 100644 --- a/packages/core/src/utils/timer.ts +++ b/packages/core/src/utils/timer.ts @@ -12,5 +12,6 @@ export function startClock() { * @returns The timestamp in milliseconds (ms). */ export function hrTimeToMs(diff: [number, number]) { - return Math.round(diff[0] * 1000 + diff[1] / 1_000) / 1_000; + const ns = diff[0] * 10 ** 9 + diff[1]; + return ns / 10 ** 6; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 26caab085..eac5b9865 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,9 +8,6 @@ patchedDependencies: detect-package-manager@3.0.1: hash: tkhlb7gk5ij4zxchvtwu3teirq path: patches/detect-package-manager@3.0.1.patch - graphql@16.8.1: - hash: 3zvcnrptpojleshpmtp6be677a - path: patches/graphql@16.8.1.patch importers: @@ -369,6 +366,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.1.3 + version: 4.1.3 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3) @@ -467,7 +467,7 @@ importers: version: 5.12.2(react@18.2.0) graphql: specifier: ^16.8.1 - version: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + version: 16.8.1 graphql-request: specifier: ^6.1.0 version: 6.1.0(graphql@16.8.1) @@ -569,6 +569,9 @@ importers: '@escape.tech/graphql-armor-max-tokens': specifier: ^2.3.0 version: 2.3.0 + '@hono/graphql-server': + specifier: ^0.4.3 + version: 0.4.3(hono@4.1.3) '@hono/node-server': specifier: ^1.8.2 version: 1.8.2 @@ -607,7 +610,7 @@ importers: version: 10.3.10 graphql: specifier: ^16.8.1 - version: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + version: 16.8.1 graphql-type-json: specifier: ^0.3.2 version: 0.3.2(graphql@16.8.1) @@ -2825,7 +2828,7 @@ packages: resolution: {integrity: sha512-h0AfPx929MWBnDlWnn/hcLHHNIAnUjws30OmyPLj9GqVmsBpj3338LELvORuuf3N1ciWI0xgkQd3NRSrmgr3ig==} engines: {node: '>=16.0.0'} dependencies: - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 optionalDependencies: '@envelop/core': 5.0.1 '@escape.tech/graphql-armor-types': 0.5.0 @@ -2835,7 +2838,7 @@ packages: resolution: {integrity: sha512-v0z2yelQL614mYFpYL/iRkieq/7H2XKbvJ6RvbGMFFSqo3eSIz8fyX0f6pyswR7myQxki4ur0MFxSn8S5jjfqw==} engines: {node: '>=16.0.0'} dependencies: - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 optionalDependencies: '@envelop/core': 4.0.3 '@escape.tech/graphql-armor-types': 0.5.0 @@ -2845,7 +2848,7 @@ packages: resolution: {integrity: sha512-4aqtUhT4ONUVWY6Z7crjPFyOK2/quUGHFU3G2+s4GYFxQHn3F5HjdI2KoY5ot2Sdijh4X+gx0ebBjUzriLNtbg==} engines: {node: '>=16.0.0'} dependencies: - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 optionalDependencies: '@envelop/core': 5.0.1 '@escape.tech/graphql-armor-types': 0.5.0 @@ -2855,7 +2858,7 @@ packages: resolution: {integrity: sha512-a7KMhb1qVHFFWw4bvGYQI637YaIZRozbfc+Fj1Vv/pwnTCJOzOgnvKO8+WBXJsFFGJ2Kj+fRORmSpz7J+lJF1w==} requiresBuild: true dependencies: - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 dev: false optional: true @@ -3060,7 +3063,7 @@ packages: hasBin: true dependencies: '@rescript/std': 9.0.0 - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 graphql-import-node: 0.0.5(graphql@16.8.1) js-yaml: 4.1.0 dev: true @@ -3125,7 +3128,7 @@ packages: '@graphql-tools/utils': 10.2.0(graphql@16.8.1) '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.1) '@repeaterjs/repeater': 3.0.5 - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 tslib: 2.6.2 value-or-promise: 1.0.12 dev: false @@ -3137,7 +3140,7 @@ packages: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: '@graphql-tools/utils': 10.2.0(graphql@16.8.1) - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 tslib: 2.6.2 dev: false @@ -3149,7 +3152,7 @@ packages: dependencies: '@graphql-tools/merge': 9.0.4(graphql@16.8.1) '@graphql-tools/utils': 10.2.0(graphql@16.8.1) - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 tslib: 2.6.2 value-or-promise: 1.0.12 dev: false @@ -3163,7 +3166,7 @@ packages: '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.1) cross-inspect: 1.0.0 dset: 3.1.3 - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 tslib: 2.6.2 dev: false @@ -3172,7 +3175,7 @@ packages: peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 dev: false /@graphql-yoga/logger@2.0.0: @@ -3212,6 +3215,16 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false + /@hono/graphql-server@0.4.3(hono@4.1.3): + resolution: {integrity: sha512-HSoOeW5F0PdlKaxC5Skc2m6mSML2yXDrVAFJfu0Hifp6igqmhqh8bhY18JgkzLyn5xuAM1/suHORETpUJnA6gQ==} + engines: {node: '>=16.0.0'} + peerDependencies: + hono: '>=3.0.0' + dependencies: + graphql: 16.8.1 + hono: 4.1.3 + dev: false + /@hono/node-server@1.8.2: resolution: {integrity: sha512-h8l2TBLCPHZBUrrkosZ6L5CpBLj6zdESyF4B+zngiCDF7aZFQJ0alVbLx7jn8PCVi9EyoFf8a4hOZFi1tD95EA==} engines: {node: '>=18.14.1'} @@ -8200,7 +8213,7 @@ packages: peerDependencies: graphql: '*' dependencies: - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 dev: true /graphql-request@6.1.0(graphql@16.8.1): @@ -8210,7 +8223,7 @@ packages: dependencies: '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.1) cross-fetch: 3.1.8 - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 transitivePeerDependencies: - encoding dev: false @@ -8220,7 +8233,7 @@ packages: peerDependencies: graphql: '>=0.8.0' dependencies: - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 dev: false /graphql-yoga@5.3.0(graphql@16.8.1): @@ -8238,7 +8251,7 @@ packages: '@whatwg-node/fetch': 0.9.17 '@whatwg-node/server': 0.9.34 dset: 3.1.3 - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 lru-cache: 10.1.0 tslib: 2.6.2 dev: false @@ -8248,10 +8261,9 @@ packages: engines: {node: '>= 10.x'} dev: true - /graphql@16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a): + /graphql@16.8.1: resolution: {integrity: sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} - patched: true /gray-matter@4.0.3: resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} @@ -9907,7 +9919,7 @@ packages: /mdast-util-from-markdown@2.0.0: resolution: {integrity: sha512-n7MTOr/z+8NAX/wmhhDji8O3bRvPTV/U0oTCaZJkjhPSKTPhS3xufVhKGF8s1pJ7Ox4QgoIU7KHseh09S+9rTA==} dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 '@types/unist': 3.0.2 decode-named-character-reference: 1.0.2 devlop: 1.1.0 @@ -9991,7 +10003,7 @@ packages: resolution: {integrity: sha512-Tl9GBNeG/AhJnQM221bJR2HPvLOSnLE/T9cJI9tlc6zwQk2nPk/4f0cHkOdEixQPC/j8UtKDdITswvLAy1OZ1w==} dependencies: '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 longest-streak: 3.1.0 mdast-util-from-markdown: 2.0.0 @@ -10018,7 +10030,7 @@ packages: dependencies: '@types/estree-jsx': 1.0.3 '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 mdast-util-from-markdown: 2.0.0 mdast-util-to-markdown: 2.1.0 @@ -10050,7 +10062,7 @@ packages: dependencies: '@types/estree-jsx': 1.0.3 '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 '@types/unist': 3.0.2 ccount: 2.0.1 devlop: 1.1.0 @@ -10094,7 +10106,7 @@ packages: dependencies: '@types/estree-jsx': 1.0.3 '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 mdast-util-from-markdown: 2.0.0 mdast-util-to-markdown: 2.1.0 @@ -10112,7 +10124,7 @@ packages: /mdast-util-phrasing@4.0.0: resolution: {integrity: sha512-xadSsJayQIucJ9n053dfQwVu1kuXg7jCTdYsMK8rqzKZh52nLfSH/k0sAxE0u+pj/zKZX+o5wB+ML5mRayOxFA==} dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 unist-util-is: 6.0.0 dev: false @@ -10159,7 +10171,7 @@ packages: /mdast-util-to-markdown@2.1.0: resolution: {integrity: sha512-SR2VnIEdVNCJbP6y7kVTJgPLifdr8WEU440fQec7qHoHOUz/oJ2jmNRqdDQ3rbiStOXb2mCDGTuwsK5OPUgYlQ==} dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 '@types/unist': 3.0.2 longest-streak: 3.1.0 mdast-util-phrasing: 4.0.0 @@ -10178,7 +10190,7 @@ packages: /mdast-util-to-string@4.0.0: resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==} dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 dev: false /mdn-data@2.0.28: @@ -10504,7 +10516,7 @@ packages: /micromark-factory-space@2.0.0: resolution: {integrity: sha512-TKr+LIDX2pkBJXFLzpyPyljzYK3MtmllMUMODTQJIUfDGncESaqB90db9IAUcz4AZAJFdd8U9zOp9ty1458rxg==} dependencies: - micromark-util-character: 2.0.1 + micromark-util-character: 2.1.0 micromark-util-types: 2.0.0 dev: false From 5a1ae49120852053b2325b07ead4377e7203efdf Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 12 Jun 2024 11:33:29 -0400 Subject: [PATCH 007/122] more custom api --- examples/reference-erc20/src/_server.ts | 2 +- packages/core/src/server/service.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/reference-erc20/src/_server.ts b/examples/reference-erc20/src/_server.ts index e285c9096..d69cd7c7c 100644 --- a/examples/reference-erc20/src/_server.ts +++ b/examples/reference-erc20/src/_server.ts @@ -1,6 +1,6 @@ import { hono } from "@/generated"; -hono.get("/kevin", (c) => { +hono.get("/router", (c) => { return c.text("kevin"); }); diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index adb0591ec..fc8f7b99b 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -34,6 +34,7 @@ export async function createServer({ let isHealthy = false; const startTime = Date.now(); + // TODO(kyle) move metrics middleware so that user requests are measured. const metricsMiddleware = createMiddleware(async (c, next) => { const commonLabels = { method: c.req.method, path: c.req.path }; common.metrics.ponder_http_server_active_requests.inc(commonLabels); @@ -135,10 +136,15 @@ export async function createServer({ return httpServer; }; + const middleware = createMiddleware(async (_, next) => { + await next(); + }); + const hono = new Hono<{ Variables: { store: ReadonlyStore; getLoader: GetLoader }; }>() .route("/_ponder", ponderApp) + .use(middleware) .route("/", app); const httpServer = await new Promise((resolve, reject) => { From 2b236576ba9bb2d76a150b8324c0e69c884d101b Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 12 Jun 2024 11:42:03 -0400 Subject: [PATCH 008/122] cleanup server --- packages/core/src/server/service.ts | 89 ++++++++++++++--------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index fc8f7b99b..64feaf83b 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -8,10 +8,9 @@ import { Hono } from "hono"; import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; -import { type GetLoader } from "../graphql/buildLoaderCache.js"; type Server = { - hono: Hono<{ Variables: { store: ReadonlyStore; getLoader: GetLoader } }>; + hono: Hono; port: number; setHealthy: () => void; kill: () => Promise; @@ -26,15 +25,41 @@ export async function createServer({ readonlyStore: ReadonlyStore; common: Common; }): Promise { - const ponderApp = new Hono<{ - Variables: { store: ReadonlyStore; getLoader: GetLoader }; - }>(); + // Create hono app - let port = common.options.port; - let isHealthy = false; const startTime = Date.now(); + let isHealthy = false; + + const ponderApp = new Hono() + .use(cors()) + .get("/metrics", async (c) => { + try { + const metrics = await common.metrics.getMetrics(); + return c.text(metrics); + } catch (error) { + return c.json(error, 500); + } + }) + .get("/health", async (c) => { + if (isHealthy) { + return c.text("", 200); + } + + const elapsed = (Date.now() - startTime) / 1000; + const max = common.options.maxHealthcheckDuration; + + if (elapsed > max) { + common.logger.warn({ + service: "server", + msg: `Historical indexing duration has exceeded the max healthcheck duration of ${max} seconds (current: ${elapsed}). Sevice is now responding as healthy and may serve incomplete data.`, + }); + return c.text("", 200); + } + + return c.text("Historical indexing is not complete.", 503); + }) + .get("/ready", async (c) => c.text("", 200)); - // TODO(kyle) move metrics middleware so that user requests are measured. const metricsMiddleware = createMiddleware(async (c, next) => { const commonLabels = { method: c.req.method, path: c.req.path }; common.metrics.ponder_http_server_active_requests.inc(commonLabels); @@ -71,36 +96,19 @@ export async function createServer({ } }); - ponderApp - .use(cors()) - .use(metricsMiddleware) - .get("/metrics", async (c) => { - try { - const metrics = await common.metrics.getMetrics(); - return c.text(metrics); - } catch (error) { - return c.json(error, 500); - } - }) - .get("/health", async (c) => { - if (isHealthy) { - return c.text("", 200); - } + const contextMiddleware = createMiddleware(async (_, next) => { + await next(); + }); - const elapsed = (Date.now() - startTime) / 1000; - const max = common.options.maxHealthcheckDuration; + const hono = new Hono() + .use(metricsMiddleware) + .route("/_ponder", ponderApp) + .use(contextMiddleware) + .route("/", app); - if (elapsed > max) { - common.logger.warn({ - service: "server", - msg: `Historical indexing duration has exceeded the max healthcheck duration of ${max} seconds (current: ${elapsed}). Sevice is now responding as healthy and may serve incomplete data.`, - }); - return c.text("", 200); - } + // Create nodejs server - return c.text("Historical indexing is not complete.", 503); - }) - .get("/ready", async (c) => c.text("", 200)); + let port = common.options.port; const createServerWithNextAvailablePort: typeof http.createServer = ( ...args: any @@ -136,17 +144,6 @@ export async function createServer({ return httpServer; }; - const middleware = createMiddleware(async (_, next) => { - await next(); - }); - - const hono = new Hono<{ - Variables: { store: ReadonlyStore; getLoader: GetLoader }; - }>() - .route("/_ponder", ponderApp) - .use(middleware) - .route("/", app); - const httpServer = await new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject(new Error("HTTP server failed to start within 5 seconds.")); From da92dcba3b6dbca5b53e02f586433ff3477e86ff Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 12 Jun 2024 11:44:11 -0400 Subject: [PATCH 009/122] more cleanup: --- packages/core/src/bin/commands/serve.ts | 2 +- packages/core/src/bin/utils/run.ts | 14 ++++++-------- packages/core/src/server/service.ts | 13 +------------ 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index 7d920795d..8c7cab216 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -104,7 +104,7 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { db: database.readonlyDb, }); - const server = await createServer({ graphqlSchema, common, readonlyStore }); + const server = await createServer({ common }); server.setHealthy(); cleanupReloadable = async () => { diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 4bf2e624b..2aa3898ca 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -94,18 +94,16 @@ export async function run({ syncStore = new PostgresSyncStore({ db: database.syncDb }); } - const readonlyStore = getReadonlyStore({ - kind: database.kind, - schema, - namespaceInfo, - db: database.readonlyDb, - }); + // const readonlyStore = getReadonlyStore({ + // kind: database.kind, + // schema, + // namespaceInfo, + // db: database.readonlyDb, + // }); const server = await createServer({ app: build.app, common, - graphqlSchema, - readonlyStore, }); // This can be a long-running operation, so it's best to do it after diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 64feaf83b..95ae6f333 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -1,30 +1,19 @@ import http from "node:http"; import type { Common } from "@/common/common.js"; -import type { ReadonlyStore } from "@/indexing-store/store.js"; import { startClock } from "@/utils/timer.js"; import { serve } from "@hono/node-server"; -import { GraphQLSchema } from "graphql"; import { Hono } from "hono"; import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; -type Server = { - hono: Hono; - port: number; - setHealthy: () => void; - kill: () => Promise; -}; - export async function createServer({ app, common, }: { app?: Hono; - graphqlSchema: GraphQLSchema; - readonlyStore: ReadonlyStore; common: Common; -}): Promise { +}) { // Create hono app const startTime = Date.now(); From 419a4617836204365da720e820b3358d7e2a8a4a Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 12 Jun 2024 12:00:55 -0400 Subject: [PATCH 010/122] hono peer dep --- package.json | 3 +- packages/core/package.json | 4 +- pnpm-lock.yaml | 101 ++++++++++++------------------------- 3 files changed, 35 insertions(+), 73 deletions(-) diff --git a/package.json b/package.json index 61d100486..26ce98d83 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "lint-staged": "^15.1.0", "simple-git-hooks": "^2.9.0", "typescript": "5.0.4", - "viem": "1.16.0" + "viem": "1.16.0", + "hono": "4.0.0" }, "lint-staged": { "*.ts": ["biome format --no-errors-on-unmatched", "biome check"], diff --git a/packages/core/package.json b/packages/core/package.json index 7a3f8b9fe..2ccafc84e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -34,7 +34,8 @@ }, "peerDependencies": { "typescript": ">=5.0.4", - "viem": ">=1.16" + "viem": ">=1.16", + "hono": ">=4" }, "peerDependenciesMeta": { "typescript": { @@ -63,7 +64,6 @@ "graphql": "^16.8.1", "graphql-type-json": "^0.3.2", "graphql-yoga": "^5.3.0", - "hono": "^4.1.3", "http-terminator": "^3.2.0", "ink": "^4.4.1", "kysely": "^0.26.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eac5b9865..7422411ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,6 +22,9 @@ importers: '@changesets/cli': specifier: ^2.26.2 version: 2.27.1 + hono: + specifier: 4.0.0 + version: 4.0.0 lint-staged: specifier: ^15.1.0 version: 15.2.0 @@ -571,7 +574,7 @@ importers: version: 2.3.0 '@hono/graphql-server': specifier: ^0.4.3 - version: 0.4.3(hono@4.1.3) + version: 0.4.3(hono@4.0.0) '@hono/node-server': specifier: ^1.8.2 version: 1.8.2 @@ -617,9 +620,6 @@ importers: graphql-yoga: specifier: ^5.3.0 version: 5.3.0(graphql@16.8.1) - hono: - specifier: ^4.1.3 - version: 4.1.3 http-terminator: specifier: ^3.2.0 version: 3.2.0 @@ -2190,7 +2190,7 @@ packages: outdent: 0.5.0 prettier: 2.8.8 resolve-from: 5.0.0 - semver: 7.5.4 + semver: 7.6.2 dev: true /@changesets/assemble-release-plan@6.0.0: @@ -2201,7 +2201,7 @@ packages: '@changesets/get-dependents-graph': 2.0.0 '@changesets/types': 6.0.0 '@manypkg/get-packages': 1.1.3 - semver: 7.5.4 + semver: 7.6.2 dev: true /@changesets/changelog-git@0.2.0: @@ -2252,7 +2252,7 @@ packages: p-limit: 2.3.0 preferred-pm: 3.1.2 resolve-from: 5.0.0 - semver: 7.5.4 + semver: 7.6.2 spawndamnit: 2.0.0 term-size: 2.2.1 tty-table: 4.2.3 @@ -2283,7 +2283,7 @@ packages: '@manypkg/get-packages': 1.1.3 chalk: 2.4.2 fs-extra: 7.0.1 - semver: 7.5.4 + semver: 7.6.2 dev: true /@changesets/get-github-info@0.5.2: @@ -2406,15 +2406,6 @@ packages: dev: false optional: true - /@envelop/core@5.0.0: - resolution: {integrity: sha512-aJdnH/ptv+cvwfvciCBe7TSvccBwo9g0S5f6u35TBVzRVqIGkK03lFlIL+x1cnfZgN9EfR2b1PH2galrT1CdCQ==} - engines: {node: '>=18.0.0'} - requiresBuild: true - dependencies: - '@envelop/types': 5.0.0 - tslib: 2.6.2 - dev: false - /@envelop/core@5.0.1: resolution: {integrity: sha512-wxA8EyE1fPnlbP0nC/SFI7uU8wSNf4YjxZhAPu0P63QbgIvqHtHsH4L3/u+rsTruzhk3OvNRgQyLsMfaR9uzAQ==} engines: {node: '>=18.0.0'} @@ -2423,7 +2414,6 @@ packages: '@envelop/types': 5.0.0 tslib: 2.6.2 dev: false - optional: true /@envelop/types@4.0.1: resolution: {integrity: sha512-ULo27/doEsP7uUhm2iTnElx13qTO6I5FKvmLoX41cpfuw8x6e0NUFknoqhEsLzAbgz8xVS5mjwcxGCXh4lDYzg==} @@ -3215,14 +3205,14 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@hono/graphql-server@0.4.3(hono@4.1.3): + /@hono/graphql-server@0.4.3(hono@4.0.0): resolution: {integrity: sha512-HSoOeW5F0PdlKaxC5Skc2m6mSML2yXDrVAFJfu0Hifp6igqmhqh8bhY18JgkzLyn5xuAM1/suHORETpUJnA6gQ==} engines: {node: '>=16.0.0'} peerDependencies: hono: '>=3.0.0' dependencies: graphql: 16.8.1 - hono: 4.1.3 + hono: 4.0.0 dev: false /@hono/node-server@1.8.2: @@ -3233,6 +3223,7 @@ packages: /@humanwhocodes/config-array@0.11.13: resolution: {integrity: sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==} engines: {node: '>=10.10.0'} + deprecated: Use @eslint/config-array instead dependencies: '@humanwhocodes/object-schema': 2.0.1 debug: 4.3.4(supports-color@8.1.1) @@ -3248,6 +3239,7 @@ packages: /@humanwhocodes/object-schema@2.0.1: resolution: {integrity: sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==} + deprecated: Use @eslint/object-schema instead dev: true /@ipld/dag-cbor@7.0.3: @@ -3701,7 +3693,7 @@ packages: natural-orderby: 2.0.3 object-treeify: 1.1.33 password-prompt: 1.1.3 - semver: 7.5.4 + semver: 7.6.2 string-width: 4.2.3 strip-ansi: 6.0.1 supports-color: 8.1.1 @@ -4351,12 +4343,6 @@ packages: '@types/unist': 2.0.10 dev: false - /@types/mdast@4.0.3: - resolution: {integrity: sha512-LsjtqsyF+d2/yFOYaN22dHZI1Cpwkrj+g06G8+qtUKlhovPW89YhqSnfKtMbkgmEtYpH2gydRNULd6y8mciAFg==} - dependencies: - '@types/unist': 3.0.2 - dev: false - /@types/mdast@4.0.4: resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} dependencies: @@ -5498,7 +5484,7 @@ packages: /builtins@5.0.1: resolution: {integrity: sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==} dependencies: - semver: 7.5.4 + semver: 7.6.2 dev: false /bundle-require@3.1.2(esbuild@0.16.17): @@ -5955,7 +5941,7 @@ packages: dot-prop: 8.0.2 env-paths: 3.0.0 json-schema-typed: 8.0.1 - semver: 7.5.4 + semver: 7.6.2 uint8array-extras: 0.3.0 dev: false @@ -8078,6 +8064,7 @@ packages: /glob@7.1.7: resolution: {integrity: sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 @@ -8089,6 +8076,7 @@ packages: /glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + deprecated: Glob versions prior to v9 are no longer supported dependencies: fs.realpath: 1.0.0 inflight: 1.0.6 @@ -8242,7 +8230,7 @@ packages: peerDependencies: graphql: ^15.2.0 || ^16.0.0 dependencies: - '@envelop/core': 5.0.0 + '@envelop/core': 5.0.1 '@graphql-tools/executor': 1.2.6(graphql@16.8.1) '@graphql-tools/schema': 10.0.3(graphql@16.8.1) '@graphql-tools/utils': 10.2.0(graphql@16.8.1) @@ -8418,24 +8406,6 @@ packages: '@types/hast': 3.0.4 dev: false - /hast-util-raw@9.0.2: - resolution: {integrity: sha512-PldBy71wO9Uq1kyaMch9AHIghtQvIwxBUkv823pKmkTM3oV1JxtsTNYdevMxvUHqcnOAuO65JKU2+0NOxc2ksA==} - dependencies: - '@types/hast': 3.0.4 - '@types/unist': 3.0.2 - '@ungap/structured-clone': 1.2.0 - hast-util-from-parse5: 8.0.1 - hast-util-to-parse5: 8.0.0 - html-void-elements: 3.0.0 - mdast-util-to-hast: 13.1.0 - parse5: 7.1.2 - unist-util-position: 5.0.0 - unist-util-visit: 5.0.0 - vfile: 6.0.1 - web-namespaces: 2.0.1 - zwitch: 2.0.4 - dev: false - /hast-util-raw@9.0.3: resolution: {integrity: sha512-ICWvVOF2fq4+7CMmtCPD5CM4QKjPbHpPotE6+8tDooV0ZuyJVUzHsrNX+O5NaRbieTf0F7FfeBOMAwi6Td0+yQ==} dependencies: @@ -8490,7 +8460,7 @@ packages: mdast-util-mdx-expression: 2.0.0 mdast-util-mdx-jsx: 3.0.0 mdast-util-mdxjs-esm: 2.0.1 - property-information: 6.4.1 + property-information: 6.5.0 space-separated-tokens: 2.0.2 style-to-object: 0.4.4 unist-util-position: 5.0.0 @@ -8576,6 +8546,10 @@ packages: minimalistic-crypto-utils: 1.0.1 dev: true + /hono@4.0.0: + resolution: {integrity: sha512-8dKhuBBpRZEodUttQhrSFJ6PQqHRjXHyeeegfxOf132pvgbf0tOb9qqb7q7eYwAWpOcYrsUOsWdJ0sQIIovhZg==} + engines: {node: '>=16.0.0'} + /hono@4.1.3: resolution: {integrity: sha512-V0I6qCw0gn2MA4LLtyXe6oD3/7ToeQf5Zv98o7uSuLuViQgWHJeYoYrZ4NbXhOtg4SaZjNJJm1+XuFB3LN+j6A==} engines: {node: '>=16.0.0'} @@ -8711,6 +8685,7 @@ packages: /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. dependencies: once: 1.4.0 wrappy: 1.0.2 @@ -9825,6 +9800,7 @@ packages: engines: {node: '>=10'} dependencies: yallist: 4.0.0 + dev: true /lucide-react@0.295.0(react@18.2.0): resolution: {integrity: sha512-5tQQ8V4Qn9DZscW55OOk9i5z4R0TfJiMjLEwM1P1jqtY5aPD3AnY049Zfb+fyXAa1JcUS5o26Wsl/3dfvTue6w==} @@ -10393,7 +10369,7 @@ packages: devlop: 1.1.0 katex: 0.16.9 micromark-factory-space: 2.0.0 - micromark-util-character: 2.0.1 + micromark-util-character: 2.1.0 micromark-util-symbol: 2.0.0 micromark-util-types: 2.0.0 dev: false @@ -10563,13 +10539,6 @@ packages: micromark-util-types: 1.1.0 dev: false - /micromark-util-character@2.0.1: - resolution: {integrity: sha512-3wgnrmEAJ4T+mGXAUfMvMAbxU9RDG43XmGce4j6CwPtVxB3vfwXSZ6KhFwDzZ3mZHhmPimMAXg71veiBGzeAZw==} - dependencies: - micromark-util-symbol: 2.0.0 - micromark-util-types: 2.0.0 - dev: false - /micromark-util-character@2.1.0: resolution: {integrity: sha512-KvOVV+X1yLBfs9dCBSopq/+G1PcgT3lAK07mC4BzXi5E7ahzMAF8oIupDDJ6mievI6F+lAATkbQQlQixJfT3aQ==} dependencies: @@ -12116,10 +12085,6 @@ packages: react-is: 16.13.1 dev: true - /property-information@6.4.1: - resolution: {integrity: sha512-OHYtXfu5aI2sS2LWFSN5rgJjrQ4pCy8i1jubJLe2QvMF8JJ++HXTUIVWFLfXJoaOfvYYjk2SN8J2wFUWIGXT4w==} - dev: false - /property-information@6.5.0: resolution: {integrity: sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==} dev: false @@ -12489,7 +12454,7 @@ packages: resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==} dependencies: '@types/hast': 3.0.4 - hast-util-raw: 9.0.2 + hast-util-raw: 9.0.3 vfile: 6.0.1 dev: false @@ -12516,7 +12481,7 @@ packages: /remark-math@6.0.0: resolution: {integrity: sha512-MMqgnP74Igy+S3WwnhQ7kqGlEerTETXMvJhrUzDikVZ2/uogJCb+WHUg97hK9/jcfc0dkD73s3LN8zU49cTEtA==} dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 mdast-util-math: 3.0.0 micromark-extension-math: 3.0.0 unified: 11.0.4 @@ -12666,6 +12631,7 @@ packages: /rimraf@2.7.1: resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true dependencies: glob: 7.2.3 @@ -12673,6 +12639,7 @@ packages: /rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true dependencies: glob: 7.2.3 @@ -12850,13 +12817,6 @@ packages: lru-cache: 6.0.0 dev: true - /semver@7.5.4: - resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} - engines: {node: '>=10'} - hasBin: true - dependencies: - lru-cache: 6.0.0 - /semver@7.6.2: resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==} engines: {node: '>=10'} @@ -14938,6 +14898,7 @@ packages: /yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: true /yaml@1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} From 526d7175b808afffc0ac6bc6360e7a235c358917 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 12 Jun 2024 12:50:53 -0400 Subject: [PATCH 011/122] wip graphql --- examples/reference-erc20/ponder-env.d.ts | 6 +++++- examples/reference-erc20/src/_server.ts | 5 ++++- packages/core/package.json | 1 - packages/core/src/build/plugin.ts | 5 ++++- packages/core/src/common/codegen.ts | 6 +++++- packages/core/src/graphql/buildGraphqlSchema.ts | 4 ++-- packages/core/src/graphql/middleware.ts | 11 +++++------ pnpm-lock.yaml | 14 +------------- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index 7c4952bd8..d1b3f4494 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -5,7 +5,7 @@ declare module "@/generated" { import type { Virtual } from "@ponder/core"; - import type { Hono } from "hono"; + import type { Hono, Context as HonoContext } from "hono"; type config = typeof import("./ponder.config.ts").default; type schema = typeof import("./ponder.schema.ts").default; @@ -14,6 +14,10 @@ declare module "@/generated" { export const hono: Hono; + export const graphQLMiddleware: () => ( + c: HonoContext, + ) => Promise; + export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< config, diff --git a/examples/reference-erc20/src/_server.ts b/examples/reference-erc20/src/_server.ts index d69cd7c7c..ea21a7a38 100644 --- a/examples/reference-erc20/src/_server.ts +++ b/examples/reference-erc20/src/_server.ts @@ -1,4 +1,7 @@ -import { hono } from "@/generated"; +import { graphQLMiddleware, hono } from "@/generated"; +import { createGraphQLMiddleware } from "@ponder/core"; + +hono.use("/graphql", graphQLMiddleware()); hono.get("/router", (c) => { return c.text("kevin"); diff --git a/packages/core/package.json b/packages/core/package.json index 2ccafc84e..651e87ad3 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -48,7 +48,6 @@ "@escape.tech/graphql-armor-max-aliases": "^2.3.0", "@escape.tech/graphql-armor-max-depth": "^2.2.0", "@escape.tech/graphql-armor-max-tokens": "^2.3.0", - "@hono/graphql-server": "^0.4.3", "@hono/node-server": "^1.8.2", "@ponder/utils": "workspace:*", "abitype": "^0.10.2", diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 017dbbd08..34e555571 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -10,6 +10,9 @@ export const ponderRegex = export const serverRegex = /^import\s+\{[^}]*\bhono\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; +export const graphqlRegex = + /^import\s+\{[^}]*\bgraphQLMiddleware\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; + export const ponderShim = `export let ponder = { fns: [], on(name, fn) { @@ -43,7 +46,7 @@ export const vitePluginPonder = (common: Common): Plugin => { transform: (code, id) => { if ( id === path.join(common.options.srcDir, SERVER_FILE) && - serverRegex.test(code) + (serverRegex.test(code) || graphqlRegex.test(code)) ) { const s = replaceStateless(code, serverRegex, serverShim); const transformed = s.toString(); diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index d6c9bc575..f7405a61c 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -11,7 +11,7 @@ export const ponderEnv = `// This file enables type checking and editor autocomp declare module "@/generated" { import type { Virtual } from "@ponder/core"; - import type { Hono } from "hono"; + import type { Hono, Context as HonoContext } from "hono"; type config = typeof import("./ponder.config.ts").default; type schema = typeof import("./ponder.schema.ts").default; @@ -20,6 +20,10 @@ declare module "@/generated" { export const hono: Hono; + export const graphQLMiddleware: () => ( + c: HonoContext, + ) => Promise; + export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< config, diff --git a/packages/core/src/graphql/buildGraphqlSchema.ts b/packages/core/src/graphql/buildGraphqlSchema.ts index c612ae9d2..bbdc92b67 100644 --- a/packages/core/src/graphql/buildGraphqlSchema.ts +++ b/packages/core/src/graphql/buildGraphqlSchema.ts @@ -1,4 +1,4 @@ -import type { IndexingStore } from "@/indexing-store/store.js"; +import type { ReadonlyStore } from "@/indexing-store/store.js"; import type { Schema } from "@/schema/common.js"; import { getTables } from "@/schema/utils.js"; import { @@ -15,7 +15,7 @@ import { buildSingularField } from "./singular.js"; // TODO(kyle) stricter type export type Parent = Record; -export type Context = { store: IndexingStore; getLoader: GetLoader }; +export type Context = { store: ReadonlyStore; getLoader: GetLoader }; export const buildGraphqlSchema = (schema: Schema): GraphQLSchema => { const queryFields: Record> = {}; diff --git a/packages/core/src/graphql/middleware.ts b/packages/core/src/graphql/middleware.ts index a9648898d..dd0e6cce4 100644 --- a/packages/core/src/graphql/middleware.ts +++ b/packages/core/src/graphql/middleware.ts @@ -1,9 +1,8 @@ import type { Schema } from "@/schema/common.js"; -import { graphqlServer } from "@hono/graphql-server"; -import { buildGraphqlSchema } from "./buildGraphqlSchema.js"; +import { createMiddleware } from "hono/factory"; -export const createGraphQLMiddleware = ({ - schema, -}: { schema: Schema }): ReturnType => { - return graphqlServer({ schema: buildGraphqlSchema(schema) }); +export const createGraphQLMiddleware = (_: { schema: Schema }) => { + return createMiddleware(async (c) => { + return c.text("graphql"); + }); }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7422411ea..f1269f9db 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -572,9 +572,6 @@ importers: '@escape.tech/graphql-armor-max-tokens': specifier: ^2.3.0 version: 2.3.0 - '@hono/graphql-server': - specifier: ^0.4.3 - version: 0.4.3(hono@4.0.0) '@hono/node-server': specifier: ^1.8.2 version: 1.8.2 @@ -3205,16 +3202,6 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@hono/graphql-server@0.4.3(hono@4.0.0): - resolution: {integrity: sha512-HSoOeW5F0PdlKaxC5Skc2m6mSML2yXDrVAFJfu0Hifp6igqmhqh8bhY18JgkzLyn5xuAM1/suHORETpUJnA6gQ==} - engines: {node: '>=16.0.0'} - peerDependencies: - hono: '>=3.0.0' - dependencies: - graphql: 16.8.1 - hono: 4.0.0 - dev: false - /@hono/node-server@1.8.2: resolution: {integrity: sha512-h8l2TBLCPHZBUrrkosZ6L5CpBLj6zdESyF4B+zngiCDF7aZFQJ0alVbLx7jn8PCVi9EyoFf8a4hOZFi1tD95EA==} engines: {node: '>=18.14.1'} @@ -8549,6 +8536,7 @@ packages: /hono@4.0.0: resolution: {integrity: sha512-8dKhuBBpRZEodUttQhrSFJ6PQqHRjXHyeeegfxOf132pvgbf0tOb9qqb7q7eYwAWpOcYrsUOsWdJ0sQIIovhZg==} engines: {node: '>=16.0.0'} + dev: true /hono@4.1.3: resolution: {integrity: sha512-V0I6qCw0gn2MA4LLtyXe6oD3/7ToeQf5Zv98o7uSuLuViQgWHJeYoYrZ4NbXhOtg4SaZjNJJm1+XuFB3LN+j6A==} From 8af92a28c0b5bfb6ec3e357495bf4f3e727ae851 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 12 Jun 2024 14:59:54 -0400 Subject: [PATCH 012/122] fix graphql --- examples/reference-erc20/ponder-env.d.ts | 8 ++-- examples/reference-erc20/src/_server.ts | 21 +++++++---- packages/core/src/bin/utils/run.ts | 14 ++++--- packages/core/src/build/plugin.ts | 10 +---- packages/core/src/common/codegen.ts | 8 ++-- packages/core/src/graphql/middleware.ts | 41 +++++++++++++++++--- packages/core/src/index.ts | 2 +- packages/core/src/server/service.ts | 48 +++++++++++++++++++++++- packages/core/src/types/virtual.ts | 12 ++++++ 9 files changed, 125 insertions(+), 39 deletions(-) diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index d1b3f4494..f61470bb3 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -12,11 +12,9 @@ declare module "@/generated" { export const ponder: Virtual.Registry; - export const hono: Hono; - - export const graphQLMiddleware: () => ( - c: HonoContext, - ) => Promise; + export const hono: Hono<{ + Variables: { db: Virtual.ReadonlyDb }; + }>; export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< diff --git a/examples/reference-erc20/src/_server.ts b/examples/reference-erc20/src/_server.ts index ea21a7a38..07a5b1584 100644 --- a/examples/reference-erc20/src/_server.ts +++ b/examples/reference-erc20/src/_server.ts @@ -1,10 +1,17 @@ -import { graphQLMiddleware, hono } from "@/generated"; -import { createGraphQLMiddleware } from "@ponder/core"; +import { hono } from "@/generated"; +import { graphQLMiddleware } from "@ponder/core"; -hono.use("/graphql", graphQLMiddleware()); +hono.use("/graphql", graphQLMiddleware); -hono.get("/router", (c) => { - return c.text("kevin"); -}); +hono.get("/router", async (c) => { + const db = c.get("db"); + const account = await db.Account.findUnique({ + id: "0x0000000000000000000000000000000000000000", + }); -export { hono }; + if (account === null) { + return c.text("Not Found!"); + } else { + return c.text(`Balance: ${account.balance.toString()}`); + } +}); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 2aa3898ca..685d2cb71 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -94,15 +94,17 @@ export async function run({ syncStore = new PostgresSyncStore({ db: database.syncDb }); } - // const readonlyStore = getReadonlyStore({ - // kind: database.kind, - // schema, - // namespaceInfo, - // db: database.readonlyDb, - // }); + const readonlyStore = getReadonlyStore({ + kind: database.kind, + schema, + namespaceInfo, + db: database.readonlyDb, + }); const server = await createServer({ app: build.app, + readonlyStore, + schema, common, }); diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 34e555571..f338a3ece 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -10,9 +10,6 @@ export const ponderRegex = export const serverRegex = /^import\s+\{[^}]*\bhono\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; -export const graphqlRegex = - /^import\s+\{[^}]*\bgraphQLMiddleware\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; - export const ponderShim = `export let ponder = { fns: [], on(name, fn) { @@ -22,12 +19,7 @@ export const ponderShim = `export let ponder = { `; export const serverShim = `import { Hono } from "hono"; -import { createGraphQLMiddleware } from "@ponder/core"; -import schema from "../ponder.schema.js"; export let hono = new Hono(); -function graphQLMiddleware() { - return createGraphQLMiddleware({ schema }); -} `; export function replaceStateless(code: string, regex: RegExp, shim: string) { @@ -46,7 +38,7 @@ export const vitePluginPonder = (common: Common): Plugin => { transform: (code, id) => { if ( id === path.join(common.options.srcDir, SERVER_FILE) && - (serverRegex.test(code) || graphqlRegex.test(code)) + serverRegex.test(code) ) { const s = replaceStateless(code, serverRegex, serverShim); const transformed = s.toString(); diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index f7405a61c..d07b90f3d 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -18,11 +18,9 @@ declare module "@/generated" { export const ponder: Virtual.Registry; - export const hono: Hono; - - export const graphQLMiddleware: () => ( - c: HonoContext, - ) => Promise; + export const hono: Hono<{ + Variables: { db: Virtual.ReadonlyDb }; + }>; export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< diff --git a/packages/core/src/graphql/middleware.ts b/packages/core/src/graphql/middleware.ts index dd0e6cce4..0e86b9ba8 100644 --- a/packages/core/src/graphql/middleware.ts +++ b/packages/core/src/graphql/middleware.ts @@ -1,8 +1,39 @@ -import type { Schema } from "@/schema/common.js"; +import type { ReadonlyStore } from "@/indexing-store/store.js"; +import { createYoga } from "graphql-yoga"; import { createMiddleware } from "hono/factory"; +import { buildGraphqlSchema } from "./buildGraphqlSchema.js"; +import { buildLoaderCache } from "./buildLoaderCache.js"; -export const createGraphQLMiddleware = (_: { schema: Schema }) => { - return createMiddleware(async (c) => { - return c.text("graphql"); +export const graphQLMiddleware = createMiddleware(async (c) => { + const db = c.get("db"); + const schema = c.get("schema"); + const graphqlSchema = buildGraphqlSchema(schema); + + const readonlyStore = db as unknown as ReadonlyStore; + + const yoga = createYoga({ + schema: graphqlSchema, + context: () => { + const getLoader = buildLoaderCache({ store: readonlyStore }); + return { store: readonlyStore, getLoader }; + }, + graphqlEndpoint: "/", + maskedErrors: process.env.NODE_ENV === "production", + logging: false, + graphiql: false, + parserAndValidationCache: false, + // plugins: [ + // maxTokensPlugin({ n: common.options.graphqlMaxOperationTokens }), + // maxDepthPlugin({ + // n: common.options.graphqlMaxOperationDepth, + // ignoreIntrospection: false, + // }), + // maxAliasesPlugin({ + // n: common.options.graphqlMaxOperationAliases, + // allowList: [], + // }), + // ], }); -}; + + return yoga.handle(c.req.raw); +}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c2956faec..294c0f2e8 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -17,4 +17,4 @@ export type NetworkConfig = Prettify; export type BlockConfig = Prettify; export type DatabaseConfig = Prettify; -export { createGraphQLMiddleware } from "@/graphql/middleware.js"; +export { graphQLMiddleware } from "@/graphql/middleware.js"; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 95ae6f333..23e6ea09c 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -1,5 +1,10 @@ import http from "node:http"; import type { Common } from "@/common/common.js"; +import type { ReadonlyStore } from "@/indexing-store/store.js"; +import type { Schema } from "@/schema/common.js"; +import { getTables } from "@/schema/utils.js"; +import type { DatabaseModel } from "@/types/model.js"; +import type { UserRecord } from "@/types/schema.js"; import { startClock } from "@/utils/timer.js"; import { serve } from "@hono/node-server"; import { Hono } from "hono"; @@ -9,9 +14,13 @@ import { createHttpTerminator } from "http-terminator"; export async function createServer({ app, + schema, + readonlyStore, common, }: { app?: Hono; + schema: Schema; + readonlyStore: ReadonlyStore; common: Common; }) { // Create hono app @@ -85,7 +94,44 @@ export async function createServer({ } }); - const contextMiddleware = createMiddleware(async (_, next) => { + const db = Object.keys(getTables(schema)).reduce<{ + [tableName: string]: Pick< + DatabaseModel, + "findUnique" | "findMany" + >; + }>((acc, tableName) => { + acc[tableName] = { + findUnique: async ({ id }) => { + common.logger.trace({ + service: "store", + msg: `${tableName}.findUnique(id=${id})`, + }); + return readonlyStore.findUnique({ + tableName, + id, + }); + }, + findMany: async ({ where, orderBy, limit, before, after } = {}) => { + common.logger.trace({ + service: "store", + msg: `${tableName}.findMany`, + }); + return readonlyStore.findMany({ + tableName, + where, + orderBy, + limit, + before, + after, + }); + }, + }; + return acc; + }, {}); + + const contextMiddleware = createMiddleware(async (c, next) => { + c.set("db", db); + c.set("schema", schema); await next(); }); diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index 18edefc39..395206445 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -157,6 +157,18 @@ export namespace Virtual { >[property], > = ([base] extends [never] ? undefined : base) | override; + export type ReadonlyDb = { + [key in keyof InferSchemaType]: Prettify< + Pick< + DatabaseModel< + // @ts-ignore + InferSchemaType[key] + >, + "findUnique" | "findMany" + > + >; + }; + export type Context< config extends Config, schema extends BuilderSchema, From d55a2a5372ff10597c4265effe75654583f72a8c Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 14:56:34 -0400 Subject: [PATCH 013/122] cleanup --- examples/reference-erc20/src/_server.ts | 2 +- packages/core/src/bin/commands/codegen.ts | 4 +- packages/core/src/bin/utils/run.test.ts | 10 +- packages/core/src/bin/utils/run.ts | 3 +- packages/core/src/build/plugin.ts | 10 +- packages/core/src/build/service.ts | 31 +- packages/core/src/common/codegen.ts | 15 +- packages/core/src/common/options.ts | 12 +- .../src/graphql/buildGraphqlSchema.test.ts | 94 ++--- .../core/src/graphql/buildGraphqlSchema.ts | 2 +- packages/core/src/graphql/middleware.test.ts | 316 +++++++++++++++ packages/core/src/graphql/middleware.ts | 86 ++-- packages/core/src/server/service.test.ts | 369 +----------------- 13 files changed, 473 insertions(+), 481 deletions(-) create mode 100644 packages/core/src/graphql/middleware.test.ts diff --git a/examples/reference-erc20/src/_server.ts b/examples/reference-erc20/src/_server.ts index 07a5b1584..d1c28ea6c 100644 --- a/examples/reference-erc20/src/_server.ts +++ b/examples/reference-erc20/src/_server.ts @@ -1,7 +1,7 @@ import { hono } from "@/generated"; import { graphQLMiddleware } from "@ponder/core"; -hono.use("/graphql", graphQLMiddleware); +hono.use("/graphql", graphQLMiddleware()); hono.get("/router", async (c) => { const db = c.get("db"); diff --git a/packages/core/src/bin/commands/codegen.ts b/packages/core/src/bin/commands/codegen.ts index 6258e3411..e867fc970 100644 --- a/packages/core/src/bin/commands/codegen.ts +++ b/packages/core/src/bin/commands/codegen.ts @@ -55,10 +55,10 @@ export async function codegen({ cliOptions }: { cliOptions: CliOptions }) { properties: { cli_command: "codegen" }, }); - runCodegen({ common }); + runCodegen({ common, graphQLSchema: buildResult.build.graphQLSchema }); logger.info({ service: "codegen", msg: "Wrote ponder-env.d.ts" }); - // logger.info({ service: "codegen", msg: "Wrote schema.graphql" }); + logger.info({ service: "codegen", msg: "Wrote schema.graphql" }); await shutdown({ reason: "Success", code: 0 }); } diff --git a/packages/core/src/bin/utils/run.test.ts b/packages/core/src/bin/utils/run.test.ts index 79485344e..dc8a6fdaf 100644 --- a/packages/core/src/bin/utils/run.test.ts +++ b/packages/core/src/bin/utils/run.test.ts @@ -5,7 +5,7 @@ import { } from "@/_test/setup.js"; import type { Build } from "@/build/index.js"; import * as codegen from "@/common/codegen.js"; -import { buildGraphqlSchema } from "@/graphql/buildGraphqlSchema.js"; +import { buildGraphQLSchema } from "@/graphql/buildGraphqlSchema.js"; import { createSchema } from "@/schema/schema.js"; import { promiseWithResolvers } from "@ponder/common"; import { beforeEach, expect, test, vi } from "vitest"; @@ -26,13 +26,13 @@ const schema = createSchema((p) => ({ }), })); -const graphqlSchema = buildGraphqlSchema(schema); +const graphQLSchema = buildGraphQLSchema(schema); test("run() kill", async (context) => { const build: Build = { buildId: "buildId", schema, - graphqlSchema, + graphQLSchema, databaseConfig: context.databaseConfig, optionsConfig: {}, networks: context.networks, @@ -62,7 +62,7 @@ test("run() setup", async (context) => { const build: Build = { buildId: "buildId", schema, - graphqlSchema, + graphQLSchema, databaseConfig: context.databaseConfig, optionsConfig: {}, networks: context.networks, @@ -91,7 +91,7 @@ test("run() setup error", async (context) => { const build: Build = { buildId: "buildId", schema, - graphqlSchema, + graphQLSchema, databaseConfig: context.databaseConfig, optionsConfig: {}, networks: context.networks, diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 4afd7e2d3..e18e2a660 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -59,6 +59,7 @@ export async function run({ optionsConfig, networks, sources, + graphQLSchema, schema, indexingFunctions, } = build; @@ -111,7 +112,7 @@ export async function run({ // starting the server so the app can become responsive more quickly. await database.migrateSyncStore(); - runCodegen({ common }); + runCodegen({ common, graphQLSchema }); // Note: can throw const syncService = await createSyncService({ diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index f338a3ece..bf6085fe2 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -1,8 +1,5 @@ -import path from "node:path"; -import type { Common } from "@/common/common.js"; import MagicString from "magic-string"; import type { Plugin } from "vite"; -import { SERVER_FILE } from "./service.js"; export const ponderRegex = /^import\s+\{[^}]*\bponder\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; @@ -32,14 +29,11 @@ export function replaceStateless(code: string, regex: RegExp, shim: string) { return s; } -export const vitePluginPonder = (common: Common): Plugin => { +export const vitePluginPonder = (): Plugin => { return { name: "ponder", transform: (code, id) => { - if ( - id === path.join(common.options.srcDir, SERVER_FILE) && - serverRegex.test(code) - ) { + if (serverRegex.test(code)) { const s = replaceStateless(code, serverRegex, serverShim); const transformed = s.toString(); const sourcemap = s.generateMap({ source: id }); diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index ea2cfb653..70caa7fe2 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -6,9 +6,11 @@ import type { Config, OptionsConfig } from "@/config/config.js"; import type { DatabaseConfig } from "@/config/database.js"; import type { Network } from "@/config/networks.js"; import type { EventSource } from "@/config/sources.js"; +import { buildGraphQLSchema } from "@/graphql/buildGraphqlSchema.js"; import type { Schema } from "@/schema/common.js"; import { glob } from "glob"; -import type { Hono } from "hono"; +import type { GraphQLSchema } from "graphql"; +import { Hono } from "hono"; import { type ViteDevServer, createServer } from "vite"; import { ViteNodeRunner } from "vite-node/client"; import { ViteNodeServer } from "vite-node/server"; @@ -25,8 +27,6 @@ import { safeBuildSchema } from "./schema.js"; import { parseViteNodeError } from "./stacktrace.js"; const BUILD_ID_VERSION = "1"; -// TODO(kyle) use option for server -export const SERVER_FILE = "_server.ts"; export type Service = { // static @@ -49,6 +49,7 @@ export type Build = { networks: Network[]; // Schema schema: Schema; + graphQLSchema: GraphQLSchema; // Indexing functions indexingFunctions: IndexingFunctions; // Server @@ -113,7 +114,7 @@ export const create = async ({ publicDir: false, customLogger: viteLogger, server: { hmr: false }, - plugins: [viteTsconfigPathsPlugin(), vitePluginPonder(common)], + plugins: [viteTsconfigPathsPlugin(), vitePluginPonder()], }); // This is Vite boilerplate (initializes the Rollup container). @@ -237,7 +238,7 @@ export const start = async ( buildService.srcRegex.test(file), ); const hasServerUpdate = invalidated.includes( - SERVER_FILE.replace(/\\/g, "/"), + common.options.serverFile.replace(/\\/g, "/"), ); // This branch could trigger if you change a `note.txt` file within `src/`. @@ -391,6 +392,8 @@ const executeIndexingFunctions = async ( for (const executeResult of executeResults) { if (executeResult.status === "error") { + console.log("Bad"); + buildService.common.logger.error({ service: "build", msg: `Error while executing '${path.relative( @@ -435,16 +438,14 @@ const executeServer = async ( } | { status: "error"; error: Error } > => { - const doesServerExist = fs.existsSync( - path.join(buildService.common.options.srcDir, SERVER_FILE), - ); + const doesServerExist = fs.existsSync(buildService.common.options.serverFile); if (doesServerExist === false) { return { status: "success" }; } const executeResult = await executeFile(buildService, { - file: path.join(buildService.common.options.srcDir, SERVER_FILE), + file: buildService.common.options.serverFile, }); if (executeResult.status === "error") { @@ -452,7 +453,7 @@ const executeServer = async ( service: "build", msg: `Error while executing '${path.relative( buildService.common.options.rootDir, - SERVER_FILE, + buildService.common.options.serverFile, )}':`, error: executeResult.error, }); @@ -462,7 +463,12 @@ const executeServer = async ( const app = executeResult.exports.hono as Hono; - // TODO: check export instanceof Hono + buildService.common.logger.debug({ + service: "build", + msg: `found server with routes: [${app.routes + .map((r) => r.path) + .join(", ")}]`, + }); return { status: "success", app }; }; @@ -489,6 +495,8 @@ const validateAndBuild = async ( common.logger[log.level]({ service: "build", msg: log.msg }); } + const graphQLSchema = buildGraphQLSchema(buildSchemaResult.schema); + // Validates and build the config const buildConfigAndIndexingFunctionsResult = await safeBuildConfigAndIndexingFunctions({ @@ -532,6 +540,7 @@ const validateAndBuild = async ( networks: buildConfigAndIndexingFunctionsResult.networks, sources: buildConfigAndIndexingFunctionsResult.sources, schema: buildSchemaResult.schema, + graphQLSchema, indexingFunctions: buildConfigAndIndexingFunctionsResult.indexingFunctions, app: rawBuild.server.app, diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index 623cdccdf..8ae0af73e 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -1,6 +1,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; import type { Common } from "@/common/common.js"; +import { type GraphQLSchema, printSchema } from "graphql"; export const ponderEnv = `// This file enables type checking and editor autocomplete for this Ponder project. // After upgrading, you may find that changes have been made to this file. @@ -38,10 +39,10 @@ declare module "@/generated" { export function runCodegen({ common, - // graphqlSchema, + graphQLSchema, }: { common: Common; - // graphqlSchema: GraphQLSchema; + graphQLSchema: GraphQLSchema; }) { writeFileSync( path.join(common.options.rootDir, "ponder-env.d.ts"), @@ -55,11 +56,11 @@ export function runCodegen({ }); mkdirSync(common.options.generatedDir, { recursive: true }); - // writeFileSync( - // path.join(common.options.generatedDir, "schema.graphql"), - // printSchema(graphqlSchema), - // "utf-8", - // ); + writeFileSync( + path.join(common.options.generatedDir, "schema.graphql"), + printSchema(graphQLSchema), + "utf-8", + ); common.logger.debug({ service: "codegen", diff --git a/packages/core/src/common/options.ts b/packages/core/src/common/options.ts index 4ae87f765..dd2205e6b 100644 --- a/packages/core/src/common/options.ts +++ b/packages/core/src/common/options.ts @@ -8,6 +8,7 @@ export type Options = { configFile: string; schemaFile: string; + serverFile: string; rootDir: string; srcDir: string; generatedDir: string; @@ -18,10 +19,6 @@ export type Options = { hostname?: string; maxHealthcheckDuration: number; - graphqlMaxOperationTokens: number; - graphqlMaxOperationDepth: number; - graphqlMaxOperationAliases: number; - telemetryUrl: string; telemetryDisabled: boolean; telemetryConfigDir: string | undefined; @@ -80,6 +77,7 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => { rootDir, configFile: path.join(rootDir, cliOptions.config), schemaFile: path.join(rootDir, "ponder.schema.ts"), + serverFile: path.join(rootDir, "src", "_server.ts"), srcDir: path.join(rootDir, "src"), generatedDir: path.join(rootDir, "generated"), ponderDir: path.join(rootDir, ".ponder"), @@ -89,12 +87,6 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => { hostname, maxHealthcheckDuration: 240, // 4 minutes - // Default limits are from Apollo: - // https://www.apollographql.com/blog/prevent-graph-misuse-with-operation-size-and-complexity-limits - graphqlMaxOperationTokens: 1000, - graphqlMaxOperationDepth: 100, - graphqlMaxOperationAliases: 30, - telemetryUrl: "https://ponder.sh/api/telemetry", telemetryDisabled: Boolean(process.env.PONDER_TELEMETRY_DISABLED), telemetryConfigDir: undefined, diff --git a/packages/core/src/graphql/buildGraphqlSchema.test.ts b/packages/core/src/graphql/buildGraphqlSchema.test.ts index 90bc09d7c..b01ecfb7d 100644 --- a/packages/core/src/graphql/buildGraphqlSchema.test.ts +++ b/packages/core/src/graphql/buildGraphqlSchema.test.ts @@ -8,7 +8,7 @@ import { createSchema } from "@/schema/schema.js"; import { encodeCheckpoint, zeroCheckpoint } from "@/utils/checkpoint.js"; import { type GraphQLType, execute, parse } from "graphql"; import { beforeEach, expect, test } from "vitest"; -import { buildGraphqlSchema } from "./buildGraphqlSchema.js"; +import { buildGraphQLSchema } from "./buildGraphqlSchema.js"; import { buildLoaderCache } from "./buildLoaderCache.js"; beforeEach(setupCommon); @@ -49,7 +49,7 @@ test("scalar", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -117,7 +117,7 @@ test("scalar list", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -185,7 +185,7 @@ test("scalar optional", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -253,7 +253,7 @@ test("scalar optional list", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -311,7 +311,7 @@ test("json", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -360,7 +360,7 @@ test("enum", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -409,7 +409,7 @@ test("enum optional", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -467,7 +467,7 @@ test("enum list", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -525,7 +525,7 @@ test("enum optional list", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -579,7 +579,7 @@ test("one", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -650,7 +650,7 @@ test("many", async (context) => { id: "0", }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -731,7 +731,7 @@ test("many w/ filter", async (context) => { id: "0", }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -786,7 +786,7 @@ test("bigint id", async (context) => { id: 0n, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -828,7 +828,7 @@ test("hex id", async (context) => { id: "0x00", }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -872,7 +872,7 @@ test("filter string eq", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -934,7 +934,7 @@ test("filter string in", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1008,7 +1008,7 @@ test("filter string contains", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1082,7 +1082,7 @@ test("filter string starts with", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1156,7 +1156,7 @@ test("filter string not ends with", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1218,7 +1218,7 @@ test("filter int eq", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1292,7 +1292,7 @@ test("filter int gt", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1354,7 +1354,7 @@ test("filter int lte", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1416,7 +1416,7 @@ test("filter int in", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1478,7 +1478,7 @@ test("filter float eq", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1552,7 +1552,7 @@ test("filter float gt", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1614,7 +1614,7 @@ test("filter float lte", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1676,7 +1676,7 @@ test("filter float in", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1738,7 +1738,7 @@ test("filter bigint eq", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1812,7 +1812,7 @@ test("filter bigint gt", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1874,7 +1874,7 @@ test("filter bigint lte", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1936,7 +1936,7 @@ test("filter bigint in", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -1998,7 +1998,7 @@ test("filer hex eq", async (context) => { await create("0", indexingStore); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2072,7 +2072,7 @@ test("filter hex gt", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2146,7 +2146,7 @@ test("filter string list eq", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2220,7 +2220,7 @@ test("filter string list has", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2285,7 +2285,7 @@ test("filter enum eq", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2340,7 +2340,7 @@ test("filter enum in", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2400,7 +2400,7 @@ test("filter ref eq", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2467,7 +2467,7 @@ test("filter ref in", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2553,7 +2553,7 @@ test("order int asc", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2637,7 +2637,7 @@ test("order bigint asc", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2721,7 +2721,7 @@ test("order bigint desc", async (context) => { }, }); - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2779,7 +2779,7 @@ test("limit default", async (context) => { await create(String(i), indexingStore); } - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2830,7 +2830,7 @@ test("limit", async (context) => { await create(String(i), indexingStore); } - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2881,7 +2881,7 @@ test("limit error", async (context) => { // await create(String(i), indexingStore); // } - const graphqlSchema = buildGraphqlSchema(schema); + const graphqlSchema = buildGraphQLSchema(schema); const document = parse(` query { @@ -2935,7 +2935,7 @@ test("filter type has correct suffixes and types", () => { }), })); - const serverSchema = buildGraphqlSchema(s); + const serverSchema = buildGraphQLSchema(s); const typeMap = serverSchema.getTypeMap(); diff --git a/packages/core/src/graphql/buildGraphqlSchema.ts b/packages/core/src/graphql/buildGraphqlSchema.ts index bbdc92b67..7d18e7685 100644 --- a/packages/core/src/graphql/buildGraphqlSchema.ts +++ b/packages/core/src/graphql/buildGraphqlSchema.ts @@ -17,7 +17,7 @@ import { buildSingularField } from "./singular.js"; export type Parent = Record; export type Context = { store: ReadonlyStore; getLoader: GetLoader }; -export const buildGraphqlSchema = (schema: Schema): GraphQLSchema => { +export const buildGraphQLSchema = (schema: Schema): GraphQLSchema => { const queryFields: Record> = {}; const { enumTypes } = buildEnumTypes({ schema }); diff --git a/packages/core/src/graphql/middleware.test.ts b/packages/core/src/graphql/middleware.test.ts new file mode 100644 index 000000000..30453bbf9 --- /dev/null +++ b/packages/core/src/graphql/middleware.test.ts @@ -0,0 +1,316 @@ +import { + setupCommon, + setupDatabaseServices, + setupIsolatedDatabase, +} from "@/_test/setup.js"; +import type { HistoricalStore, ReadonlyStore } from "@/indexing-store/store.js"; +import type { Schema } from "@/schema/common.js"; +import { createSchema } from "@/schema/schema.js"; +import { encodeCheckpoint, zeroCheckpoint } from "@/utils/checkpoint.js"; +import { Hono } from "hono"; +import { createMiddleware } from "hono/factory"; +import { beforeEach, expect, test } from "vitest"; +import { graphQLMiddleware } from "./middleware.js"; + +beforeEach(setupCommon); +beforeEach(setupIsolatedDatabase); + +const contextMiddleware = (schema: Schema, readonlyStore: ReadonlyStore) => + createMiddleware(async (c, next) => { + c.set("readonlyStore", readonlyStore); + c.set("schema", schema); + await next(); + }); + +test("graphQLMiddleware serves request", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.string(), + string: p.string(), + int: p.int(), + float: p.float(), + boolean: p.boolean(), + hex: p.hex(), + bigint: p.bigint(), + }), + })); + + const { indexingStore, readonlyStore, cleanup } = await setupDatabaseServices( + context, + { schema }, + ); + + await indexingStore.create({ + tableName: "table", + encodedCheckpoint: encodeCheckpoint(zeroCheckpoint), + id: "0", + data: { + string: "0", + int: 0, + float: 0, + boolean: false, + hex: "0x0", + bigint: 0n, + }, + }); + + await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); + + const app = new Hono() + .use(contextMiddleware(schema, readonlyStore)) + .use("/graphql", graphQLMiddleware()); + + const response = await app.request("/graphql", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + query: ` + query { + table(id: "0") { + id + string + int + float + boolean + hex + bigint + } + } + `, + }), + }); + + expect(response.status).toBe(200); + + expect(await response.json()).toMatchObject({ + data: { + table: { + id: "0", + string: "0", + int: 0, + float: 0, + boolean: false, + hex: "0x00", + bigint: "0", + }, + }, + }); + + await cleanup(); +}); + +test("graphQLMiddleware throws error when extra filter is applied", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.string(), + string: p.string(), + int: p.int(), + float: p.float(), + boolean: p.boolean(), + hex: p.hex(), + bigint: p.bigint(), + }), + })); + + const { readonlyStore, cleanup } = await setupDatabaseServices(context, { + schema, + }); + + const app = new Hono() + .use(contextMiddleware(schema, readonlyStore)) + .use("/graphql", graphQLMiddleware()); + + const response = await app.request("/graphql", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + query: ` + { + table(id: "0", doesntExist: "kevin") { + id + string + int + float + boolean + hex + bigint + } + } + `, + }), + }); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.errors[0].message).toBe( + 'Unknown argument "doesntExist" on field "Query.table".', + ); + + await cleanup(); +}); + +test("graphQLMiddleware throws error for token limit", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ id: p.string() }), + })); + + const { readonlyStore, cleanup } = await setupDatabaseServices(context, { + schema, + }); + + const app = new Hono() + .use(contextMiddleware(schema, readonlyStore)) + .use("/graphql", graphQLMiddleware({ maxOperationTokens: 3 })); + + const response = await app.request("/graphql", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + query: ` + { + __schema { + types { + fields { + type { + fields { + type { + description + } + } + } + } + } + } + } + `, + }), + }); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.errors[0].message).toBe( + "Syntax Error: Token limit of 3 exceeded.", + ); + + await cleanup(); +}); + +test("graphQLMiddleware throws error for depth limit", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ id: p.string() }), + })); + + const { readonlyStore, cleanup } = await setupDatabaseServices(context, { + schema, + }); + + const app = new Hono() + .use(contextMiddleware(schema, readonlyStore)) + .use("/graphql", graphQLMiddleware({ maxOperationDepth: 5 })); + + const response = await app.request("/graphql", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + query: ` + { + __schema { + types { + fields { + type { + fields { + type { + description + } + } + } + } + } + } + } + `, + }), + }); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.errors[0].message).toBe( + "Syntax Error: Query depth limit of 5 exceeded, found 7.", + ); + + await cleanup(); +}); + +test("graphQLMiddleware throws error for max aliases", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ id: p.string() }), + })); + + const { readonlyStore, cleanup } = await setupDatabaseServices(context, { + schema, + }); + + const app = new Hono() + .use(contextMiddleware(schema, readonlyStore)) + .use("/graphql", graphQLMiddleware({ maxOperationAliases: 2 })); + + const response = await app.request("/graphql", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + query: ` + { + __schema { + types { + fields { + type { + alias1: fields { + type { + description + } + } + alias2: fields { + type { + description + } + } + alias3: fields { + type { + description + } + } + } + } + } + } + } + `, + }), + }); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.errors[0].message).toBe( + "Syntax Error: Aliases limit of 2 exceeded, found 3.", + ); + + await cleanup(); +}); + +test("graphQLMiddleware interactive", async (context) => { + const { readonlyStore, cleanup } = await setupDatabaseServices(context, { + schema: {}, + }); + + const app = new Hono() + .use(contextMiddleware({}, readonlyStore)) + .use("/graphql", graphQLMiddleware({ maxOperationAliases: 2 })); + + const response = await app.request("/graphql"); + + expect(response.status).toBe(200); + + await cleanup(); +}); diff --git a/packages/core/src/graphql/middleware.ts b/packages/core/src/graphql/middleware.ts index ad9c8e671..b50d5c7e1 100644 --- a/packages/core/src/graphql/middleware.ts +++ b/packages/core/src/graphql/middleware.ts @@ -1,39 +1,61 @@ -import type { ReadonlyStore } from "@/indexing-store/store.js"; +import { graphiQLHtml } from "@/ui/graphiql.html.js"; +import { maxAliasesPlugin } from "@escape.tech/graphql-armor-max-aliases"; +import { maxDepthPlugin } from "@escape.tech/graphql-armor-max-depth"; +import { maxTokensPlugin } from "@escape.tech/graphql-armor-max-tokens"; import { createYoga } from "graphql-yoga"; import { createMiddleware } from "hono/factory"; -import { buildGraphqlSchema } from "./buildGraphqlSchema.js"; +import { buildGraphQLSchema } from "./buildGraphqlSchema.js"; import { buildLoaderCache } from "./buildLoaderCache.js"; -export const graphQLMiddleware = createMiddleware(async (c) => { - const db = c.get("db"); - const schema = c.get("schema"); - const graphqlSchema = buildGraphqlSchema(schema); +export const graphQLMiddleware = ( + { + maxOperationTokens = 1000, + maxOperationDepth = 100, + maxOperationAliases = 30, + }: { + maxOperationTokens?: number; + maxOperationDepth?: number; + maxOperationAliases?: number; + } = { + // Default limits are from Apollo: + // https://www.apollographql.com/blog/prevent-graph-misuse-with-operation-size-and-complexity-limit + maxOperationTokens: 1000, + maxOperationDepth: 100, + maxOperationAliases: 30, + }, +) => + createMiddleware(async (c) => { + const readonlyStore = c.get("readonlyStore"); + const schema = c.get("schema"); + const graphqlSchema = buildGraphQLSchema(schema); - const readonlyStore = db as unknown as ReadonlyStore; + if (c.req.method === "GET") { + return c.html(graphiQLHtml(c.req.path)); + } - const yoga = createYoga({ - schema: graphqlSchema, - context: () => { - const getLoader = buildLoaderCache({ store: readonlyStore }); - return { store: readonlyStore, getLoader }; - }, - graphqlEndpoint: "/", - maskedErrors: process.env.NODE_ENV === "production", - logging: false, - graphiql: false, - parserAndValidationCache: false, - // plugins: [ - // maxTokensPlugin({ n: common.options.graphqlMaxOperationTokens }), - // maxDepthPlugin({ - // n: common.options.graphqlMaxOperationDepth, - // ignoreIntrospection: false, - // }), - // maxAliasesPlugin({ - // n: common.options.graphqlMaxOperationAliases, - // allowList: [], - // }), - // ], - }); + const yoga = createYoga({ + schema: graphqlSchema, + context: () => { + const getLoader = buildLoaderCache({ store: readonlyStore }); + return { store: readonlyStore, getLoader }; + }, + graphqlEndpoint: c.req.path, + maskedErrors: process.env.NODE_ENV === "production", + logging: false, + graphiql: false, + parserAndValidationCache: false, + plugins: [ + maxTokensPlugin({ n: maxOperationTokens }), + maxDepthPlugin({ + n: maxOperationDepth, + ignoreIntrospection: false, + }), + maxAliasesPlugin({ + n: maxOperationAliases, + allowList: [], + }), + ], + }); - return yoga.handle(c.req.raw); -}) as any; + return yoga.handle(c.req.raw); + }) as any; diff --git a/packages/core/src/server/service.test.ts b/packages/core/src/server/service.test.ts index f87b0b74a..dd053f0c7 100644 --- a/packages/core/src/server/service.test.ts +++ b/packages/core/src/server/service.test.ts @@ -1,14 +1,7 @@ -import { - setupCommon, - setupDatabaseServices, - setupIsolatedDatabase, -} from "@/_test/setup.js"; -import type { HistoricalStore, ReadonlyStore } from "@/indexing-store/store.js"; -import { createSchema } from "@/schema/schema.js"; -import { encodeCheckpoint, zeroCheckpoint } from "@/utils/checkpoint.js"; -import type { GraphQLSchema } from "graphql"; +import { setupCommon, setupIsolatedDatabase } from "@/_test/setup.js"; +import type { ReadonlyStore } from "@/indexing-store/store.js"; +import type { Schema } from "@/schema/common.js"; import { beforeEach, expect, test, vi } from "vitest"; -import { buildGraphqlSchema } from "../graphql/buildGraphqlSchema.js"; import { createServer } from "./service.js"; beforeEach(setupCommon); @@ -16,13 +9,13 @@ beforeEach(setupIsolatedDatabase); test("port", async (context) => { const server1 = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: context.common, readonlyStore: {} as ReadonlyStore, }); const server2 = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: context.common, readonlyStore: {} as ReadonlyStore, }); @@ -35,7 +28,7 @@ test("port", async (context) => { test("not healthy", async (context) => { const server = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: { ...context.common, options: { ...context.common.options, maxHealthcheckDuration: 5 }, @@ -52,7 +45,7 @@ test("not healthy", async (context) => { test("healthy", async (context) => { const server = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: { ...context.common, options: { ...context.common.options, maxHealthcheckDuration: 0 }, @@ -69,7 +62,7 @@ test("healthy", async (context) => { test("healthy PUT", async (context) => { const server = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: { ...context.common, options: { ...context.common.options, maxHealthcheckDuration: 0 }, @@ -86,7 +79,7 @@ test("healthy PUT", async (context) => { test("metrics", async (context) => { const server = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: context.common, readonlyStore: {} as ReadonlyStore, }); @@ -100,7 +93,7 @@ test("metrics", async (context) => { test("metrics error", async (context) => { const server = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: context.common, readonlyStore: {} as ReadonlyStore, }); @@ -117,7 +110,7 @@ test("metrics error", async (context) => { test("metrics PUT", async (context) => { const server = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: context.common, readonlyStore: {} as ReadonlyStore, }); @@ -129,345 +122,9 @@ test("metrics PUT", async (context) => { await server.kill(); }); -test("graphql", async (context) => { - const schema = createSchema((p) => ({ - table: p.createTable({ - id: p.string(), - string: p.string(), - int: p.int(), - float: p.float(), - boolean: p.boolean(), - hex: p.hex(), - bigint: p.bigint(), - }), - })); - - const { indexingStore, readonlyStore, cleanup } = await setupDatabaseServices( - context, - { schema }, - ); - - await indexingStore.create({ - tableName: "table", - encodedCheckpoint: encodeCheckpoint(zeroCheckpoint), - id: "0", - data: { - string: "0", - int: 0, - float: 0, - boolean: false, - hex: "0x0", - bigint: 0n, - }, - }); - - await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); - - const graphqlSchema = buildGraphqlSchema(schema); - - const server = await createServer({ - graphqlSchema: graphqlSchema, - common: context.common, - readonlyStore: readonlyStore, - }); - server.setHealthy(); - - const response = await server.hono.request("/graphql", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - query: ` - query { - table(id: "0") { - id - string - int - float - boolean - hex - bigint - } - } - `, - }), - }); - - expect(response.status).toBe(200); - - expect(await response.json()).toMatchObject({ - data: { - table: { - id: "0", - string: "0", - int: 0, - float: 0, - boolean: false, - hex: "0x00", - bigint: "0", - }, - }, - }); - - await cleanup(); - - await server.kill(); -}); - -test("graphql extra filter", async (context) => { - const schema = createSchema((p) => ({ - table: p.createTable({ - id: p.string(), - string: p.string(), - int: p.int(), - float: p.float(), - boolean: p.boolean(), - hex: p.hex(), - bigint: p.bigint(), - }), - })); - - const { readonlyStore, cleanup } = await setupDatabaseServices(context, { - schema, - }); - - const graphqlSchema = buildGraphqlSchema(schema); - - const server = await createServer({ - graphqlSchema: graphqlSchema, - common: context.common, - readonlyStore: readonlyStore, - }); - server.setHealthy(); - - const response = await server.hono.request("/graphql", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - query: ` - { - table(id: "0", doesntExist: "kevin") { - id - string - int - float - boolean - hex - bigint - } - } - `, - }), - }); - - expect(response.status).toBe(200); - const body = await response.json(); - expect(body.errors[0].message).toBe( - 'Unknown argument "doesntExist" on field "Query.table".', - ); - - await cleanup(); - - await server.kill(); -}); - -test("graphql token limit error", async (context) => { - const schema = createSchema((p) => ({ - table: p.createTable({ id: p.string() }), - })); - - const { readonlyStore, cleanup } = await setupDatabaseServices(context, { - schema, - }); - - const graphqlSchema = buildGraphqlSchema(schema); - - const server = await createServer({ - graphqlSchema: graphqlSchema, - common: { - ...context.common, - options: { ...context.common.options, graphqlMaxOperationTokens: 3 }, - }, - readonlyStore: readonlyStore, - }); - server.setHealthy(); - - const response = await server.hono.request("/graphql", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - query: ` - { - __schema { - types { - fields { - type { - fields { - type { - description - } - } - } - } - } - } - } - `, - }), - }); - - expect(response.status).toBe(200); - const body = await response.json(); - expect(body.errors[0].message).toBe( - "Syntax Error: Token limit of 3 exceeded.", - ); - - await cleanup(); - - await server.kill(); -}); - -test("graphql depth limit error", async (context) => { - const schema = createSchema((p) => ({ - table: p.createTable({ id: p.string() }), - })); - - const { readonlyStore, cleanup } = await setupDatabaseServices(context, { - schema, - }); - - const graphqlSchema = buildGraphqlSchema(schema); - - const server = await createServer({ - graphqlSchema: graphqlSchema, - common: { - ...context.common, - options: { ...context.common.options, graphqlMaxOperationDepth: 5 }, - }, - readonlyStore: readonlyStore, - }); - server.setHealthy(); - - const response = await server.hono.request("/graphql", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - query: ` - { - __schema { - types { - fields { - type { - fields { - type { - description - } - } - } - } - } - } - } - `, - }), - }); - - expect(response.status).toBe(200); - const body = await response.json(); - expect(body.errors[0].message).toBe( - "Syntax Error: Query depth limit of 5 exceeded, found 7.", - ); - - await cleanup(); - - await server.kill(); -}); - -test("graphql max aliases error", async (context) => { - const schema = createSchema((p) => ({ - table: p.createTable({ id: p.string() }), - })); - - const { readonlyStore, cleanup } = await setupDatabaseServices(context, { - schema, - }); - - const graphqlSchema = buildGraphqlSchema(schema); - - const server = await createServer({ - graphqlSchema: graphqlSchema, - common: { - ...context.common, - options: { ...context.common.options, graphqlMaxOperationAliases: 2 }, - }, - readonlyStore: readonlyStore, - }); - server.setHealthy(); - - const response = await server.hono.request("/graphql", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - query: ` - { - __schema { - types { - fields { - type { - alias1: fields { - type { - description - } - } - alias2: fields { - type { - description - } - } - alias3: fields { - type { - description - } - } - } - } - } - } - } - `, - }), - }); - - expect(response.status).toBe(200); - const body = await response.json(); - expect(body.errors[0].message).toBe( - "Syntax Error: Aliases limit of 2 exceeded, found 3.", - ); - - await cleanup(); - - await server.kill(); -}); - -test("graphql interactive", async (context) => { - const server = await createServer({ - graphqlSchema: {} as GraphQLSchema, - common: context.common, - readonlyStore: {} as ReadonlyStore, - }); - server.setHealthy(); - - const response = await server.hono.request("/graphql"); - - expect(response.status).toBe(200); - - await server.kill(); -}); - test("missing route", async (context) => { const server = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: context.common, readonlyStore: {} as ReadonlyStore, }); @@ -483,7 +140,7 @@ test("missing route", async (context) => { // create a socket connection, it just calls the request handler function directly. test.skip("kill", async (context) => { const server = await createServer({ - graphqlSchema: {} as GraphQLSchema, + schema: {} as Schema, common: context.common, readonlyStore: {} as ReadonlyStore, }); From 63d3a78db0b4da405c1261269ef0ac6f141d457d Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 15:32:57 -0400 Subject: [PATCH 014/122] more cleanup --- packages/core/src/build/service.ts | 2 +- packages/core/src/graphql/middleware.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 70caa7fe2..47018fb7b 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -465,7 +465,7 @@ const executeServer = async ( buildService.common.logger.debug({ service: "build", - msg: `found server with routes: [${app.routes + msg: `Detected a custom server with routes: [${app.routes .map((r) => r.path) .join(", ")}]`, }); diff --git a/packages/core/src/graphql/middleware.ts b/packages/core/src/graphql/middleware.ts index b50d5c7e1..f2873064d 100644 --- a/packages/core/src/graphql/middleware.ts +++ b/packages/core/src/graphql/middleware.ts @@ -58,4 +58,4 @@ export const graphQLMiddleware = ( }); return yoga.handle(c.req.raw); - }) as any; + }); From 56fd057bd4dd64d98a6af4a4f891b53036cad6ba Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 16:13:04 -0400 Subject: [PATCH 015/122] . --- .../{src/_server.ts => ponder.server.ts} | 0 .../core/src/_test/e2e/erc20/src/_server.ts | 4 ++++ .../core/src/_test/e2e/factory/src/_server.ts | 4 ++++ packages/core/src/_test/utils.ts | 2 +- packages/core/src/build/service.ts | 7 ++++++- packages/core/src/common/options.ts | 2 +- .../src/graphql/buildGraphqlSchema.test.ts | 2 +- packages/core/src/server/service.test.ts | 18 +++++++++++------- packages/core/src/server/service.ts | 1 + pnpm-lock.yaml | 6 +++--- 10 files changed, 32 insertions(+), 14 deletions(-) rename examples/reference-erc20/{src/_server.ts => ponder.server.ts} (100%) create mode 100644 packages/core/src/_test/e2e/erc20/src/_server.ts create mode 100644 packages/core/src/_test/e2e/factory/src/_server.ts diff --git a/examples/reference-erc20/src/_server.ts b/examples/reference-erc20/ponder.server.ts similarity index 100% rename from examples/reference-erc20/src/_server.ts rename to examples/reference-erc20/ponder.server.ts diff --git a/packages/core/src/_test/e2e/erc20/src/_server.ts b/packages/core/src/_test/e2e/erc20/src/_server.ts new file mode 100644 index 000000000..613324c36 --- /dev/null +++ b/packages/core/src/_test/e2e/erc20/src/_server.ts @@ -0,0 +1,4 @@ +import { hono } from "@/generated"; +import { graphQLMiddleware } from "@/index.js"; + +hono.use("/graphql", graphQLMiddleware()); diff --git a/packages/core/src/_test/e2e/factory/src/_server.ts b/packages/core/src/_test/e2e/factory/src/_server.ts new file mode 100644 index 000000000..613324c36 --- /dev/null +++ b/packages/core/src/_test/e2e/factory/src/_server.ts @@ -0,0 +1,4 @@ +import { hono } from "@/generated"; +import { graphQLMiddleware } from "@/index.js"; + +hono.use("/graphql", graphQLMiddleware()); diff --git a/packages/core/src/_test/utils.ts b/packages/core/src/_test/utils.ts index 72c420fc5..ae1a03bef 100644 --- a/packages/core/src/_test/utils.ts +++ b/packages/core/src/_test/utils.ts @@ -689,7 +689,7 @@ export async function waitForHealthy(port: number) { reject(new Error("Timed out while waiting for app to become healthy.")); }, 5_000); const interval = setInterval(async () => { - const response = await fetch(`http://localhost:${port}/health`); + const response = await fetch(`http://localhost:${port}/_ponder/health`); if (response.status === 200) { clearTimeout(timeout); clearInterval(interval); diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 47018fb7b..cf11cd366 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -243,7 +243,12 @@ export const start = async ( // This branch could trigger if you change a `note.txt` file within `src/`. // Note: We could probably do a better job filtering out files in `isFileIgnored`. - if (!hasConfigUpdate && !hasSchemaUpdate && !hasIndexingFunctionUpdate) { + if ( + !hasConfigUpdate && + !hasSchemaUpdate && + !hasIndexingFunctionUpdate && + !hasServerUpdate + ) { return; } diff --git a/packages/core/src/common/options.ts b/packages/core/src/common/options.ts index dd2205e6b..fef3b4271 100644 --- a/packages/core/src/common/options.ts +++ b/packages/core/src/common/options.ts @@ -77,7 +77,7 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => { rootDir, configFile: path.join(rootDir, cliOptions.config), schemaFile: path.join(rootDir, "ponder.schema.ts"), - serverFile: path.join(rootDir, "src", "_server.ts"), + serverFile: path.join(rootDir, "ponder.server.ts"), srcDir: path.join(rootDir, "src"), generatedDir: path.join(rootDir, "generated"), ponderDir: path.join(rootDir, ".ponder"), diff --git a/packages/core/src/graphql/buildGraphqlSchema.test.ts b/packages/core/src/graphql/buildGraphqlSchema.test.ts index b01ecfb7d..1e3a12e62 100644 --- a/packages/core/src/graphql/buildGraphqlSchema.test.ts +++ b/packages/core/src/graphql/buildGraphqlSchema.test.ts @@ -290,7 +290,7 @@ test("scalar optional list", async (context) => { await cleanup(); }); -test("json", async (context) => { +test.only("json", async (context) => { const schema = createSchema((p) => ({ table: p.createTable({ id: p.string(), diff --git a/packages/core/src/server/service.test.ts b/packages/core/src/server/service.test.ts index dd053f0c7..363ee1945 100644 --- a/packages/core/src/server/service.test.ts +++ b/packages/core/src/server/service.test.ts @@ -36,7 +36,7 @@ test("not healthy", async (context) => { readonlyStore: {} as ReadonlyStore, }); - const response = await server.hono.request("/health"); + const response = await server.hono.request("/_ponder/health"); expect(response.status).toBe(503); @@ -53,7 +53,7 @@ test("healthy", async (context) => { readonlyStore: {} as ReadonlyStore, }); - const response = await server.hono.request("/health"); + const response = await server.hono.request("/_ponder/health"); expect(response.status).toBe(200); @@ -70,7 +70,9 @@ test("healthy PUT", async (context) => { readonlyStore: {} as ReadonlyStore, }); - const response = await server.hono.request("/health", { method: "PUT" }); + const response = await server.hono.request("/_ponder/health", { + method: "PUT", + }); expect(response.status).toBe(404); @@ -84,7 +86,7 @@ test("metrics", async (context) => { readonlyStore: {} as ReadonlyStore, }); - const response = await server.hono.request("/metrics"); + const response = await server.hono.request("/_ponder/metrics"); expect(response.status).toBe(200); @@ -101,7 +103,7 @@ test("metrics error", async (context) => { const metricsSpy = vi.spyOn(context.common.metrics, "getMetrics"); metricsSpy.mockRejectedValueOnce(new Error()); - const response = await server.hono.request("/metrics"); + const response = await server.hono.request("/_ponder/metrics"); expect(response.status).toBe(500); @@ -115,7 +117,9 @@ test("metrics PUT", async (context) => { readonlyStore: {} as ReadonlyStore, }); - const response = await server.hono.request("/metrics", { method: "PUT" }); + const response = await server.hono.request("/_ponder/metrics", { + method: "PUT", + }); expect(response.status).toBe(404); @@ -147,5 +151,5 @@ test.skip("kill", async (context) => { await server.kill(); - expect(() => server.hono.request("/health")).rejects.toThrow(); + expect(() => server.hono.request("/_ponder/health")).rejects.toThrow(); }); diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 23e6ea09c..c42e75845 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -131,6 +131,7 @@ export async function createServer({ const contextMiddleware = createMiddleware(async (c, next) => { c.set("db", db); + c.set("readonlyStore", readonlyStore); c.set("schema", schema); await next(); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a1ef67d08..1516ef739 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -443,7 +443,7 @@ importers: dependencies: forge-std: specifier: github:foundry-rs/forge-std - version: github.com/foundry-rs/forge-std/52715a217dc51d0de15877878ab8213f6cbbbab5 + version: github.com/foundry-rs/forge-std/6e05729b76f1ae0d437e74951aef1ca987788ab3 examples/with-foundry/ponder: dependencies: @@ -14980,8 +14980,8 @@ packages: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} dev: false - github.com/foundry-rs/forge-std/52715a217dc51d0de15877878ab8213f6cbbbab5: - resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/52715a217dc51d0de15877878ab8213f6cbbbab5} + github.com/foundry-rs/forge-std/6e05729b76f1ae0d437e74951aef1ca987788ab3: + resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/6e05729b76f1ae0d437e74951aef1ca987788ab3} name: forge-std version: 1.8.2 dev: false From bbb1eab2bd29b38a8d601da3822b527723a2b048 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 16:53:44 -0400 Subject: [PATCH 016/122] fix graphql json --- package.json | 1 + packages/core/package.json | 17 +- .../src/graphql/buildGraphqlSchema.test.ts | 2 +- packages/core/src/graphql/entity.ts | 2 +- packages/core/src/graphql/graphQLJson.ts | 63 +++ pnpm-lock.yaml | 467 +++++++++++++----- 6 files changed, 410 insertions(+), 142 deletions(-) create mode 100644 packages/core/src/graphql/graphQLJson.ts diff --git a/package.json b/package.json index 26ce98d83..29b9b15ea 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "packageManager": "pnpm@8.6.10", "pnpm": { "patchedDependencies": { + "graphql@16.8.1": "patches/graphql@16.8.1.patch", "detect-package-manager@3.0.1": "patches/detect-package-manager@3.0.1.patch" }, "peerDependencyRules": { diff --git a/packages/core/package.json b/packages/core/package.json index d9eaec8fb..3c5cf7360 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -33,9 +33,9 @@ "typecheck": "tsc --noEmit" }, "peerDependencies": { + "hono": ">=4", "typescript": ">=5.0.4", - "viem": ">=1.16", - "hono": ">=4" + "viem": ">=1.16" }, "peerDependenciesMeta": { "typescript": { @@ -45,10 +45,10 @@ "dependencies": { "@babel/code-frame": "^7.23.4", "@commander-js/extra-typings": "^12.0.1", - "@escape.tech/graphql-armor-max-aliases": "^2.3.0", - "@escape.tech/graphql-armor-max-depth": "^2.2.0", - "@escape.tech/graphql-armor-max-tokens": "^2.3.0", - "@hono/node-server": "^1.11.2", + "@escape.tech/graphql-armor-max-aliases": "^2.4.0", + "@escape.tech/graphql-armor-max-depth": "^2.3.0", + "@escape.tech/graphql-armor-max-tokens": "^2.4.0", + "@hono/node-server": "^1.11.3", "@ponder/utils": "workspace:*", "abitype": "^0.10.2", "better-sqlite3": "^10.0.0", @@ -60,9 +60,8 @@ "emittery": "^1.0.1", "ethereum-bloom-filters": "^1.0.10", "glob": "^10.3.10", - "graphql": "^16.8.1", - "graphql-type-json": "^0.3.2", - "graphql-yoga": "^5.3.0", + "graphql": "^16.8.2", + "graphql-yoga": "^5.3.1", "http-terminator": "^3.2.0", "ink": "^4.4.1", "kysely": "^0.26.3", diff --git a/packages/core/src/graphql/buildGraphqlSchema.test.ts b/packages/core/src/graphql/buildGraphqlSchema.test.ts index 1e3a12e62..b01ecfb7d 100644 --- a/packages/core/src/graphql/buildGraphqlSchema.test.ts +++ b/packages/core/src/graphql/buildGraphqlSchema.test.ts @@ -290,7 +290,7 @@ test("scalar optional list", async (context) => { await cleanup(); }); -test.only("json", async (context) => { +test("json", async (context) => { const schema = createSchema((p) => ({ table: p.createTable({ id: p.string(), diff --git a/packages/core/src/graphql/entity.ts b/packages/core/src/graphql/entity.ts index ed2c15884..9f4177566 100644 --- a/packages/core/src/graphql/entity.ts +++ b/packages/core/src/graphql/entity.ts @@ -23,9 +23,9 @@ import { GraphQLObjectType, GraphQLString, } from "graphql"; -import { GraphQLJSON } from "graphql-type-json"; import type { Context, Parent } from "./buildGraphqlSchema.js"; import { buildWhereObject } from "./filter.js"; +import { GraphQLJSON } from "./graphQLJson.js"; import type { PluralResolver } from "./plural.js"; import { SCALARS } from "./scalar.js"; diff --git a/packages/core/src/graphql/graphQLJson.ts b/packages/core/src/graphql/graphQLJson.ts new file mode 100644 index 000000000..b727eceaf --- /dev/null +++ b/packages/core/src/graphql/graphQLJson.ts @@ -0,0 +1,63 @@ +import { + type GraphQLScalarLiteralParser, + GraphQLScalarType, + Kind, + type ObjectValueNode, + type ValueNode, + print, +} from "graphql"; + +// Modified from https://github.com/taion/graphql-type-json/blob/master/src/index.js + +export const GraphQLJSON = new GraphQLScalarType({ + name: "JSON", + description: + "The `JSON` scalar type represents JSON values as specified by [ECMA-404](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf).", + serialize: (x) => x, + parseValue: (x) => x, + parseLiteral: (ast, variables) => { + if (ast.kind !== Kind.OBJECT) { + throw new TypeError( + `JSONObject cannot represent non-object value: ${print(ast)}`, + ); + } + + return parseObject(ast, variables); + }, +}); + +const parseLiteral = ( + ast: ValueNode, + variables: Parameters[1], +): ReturnType> => { + switch (ast.kind) { + case Kind.STRING: + case Kind.BOOLEAN: + return ast.value; + case Kind.INT: + case Kind.FLOAT: + return parseFloat(ast.value); + case Kind.OBJECT: + return parseObject(ast, variables); + case Kind.LIST: + return ast.values.map((n) => parseLiteral(n, variables)); + case Kind.NULL: + return null; + case Kind.VARIABLE: + return variables ? variables[ast.name.value] : undefined; + default: + throw new TypeError(`JSON cannot represent value: ${print(ast)}`); + } +}; + +const parseObject = ( + ast: ObjectValueNode, + variables: Parameters[1], +) => { + const value = Object.create(null); + ast.fields.forEach((field) => { + value[field.name.value] = parseLiteral(field.value, variables); + }); + + return value; +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1516ef739..b851e9506 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ patchedDependencies: detect-package-manager@3.0.1: hash: tkhlb7gk5ij4zxchvtwu3teirq path: patches/detect-package-manager@3.0.1.patch + graphql@16.8.1: + hash: 3zvcnrptpojleshpmtp6be677a + path: patches/graphql@16.8.1.patch importers: @@ -476,7 +479,7 @@ importers: version: 5.12.2(react@18.2.0) graphql: specifier: ^16.8.1 - version: 16.8.1 + version: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) graphql-request: specifier: ^6.1.0 version: 6.1.0(graphql@16.8.1) @@ -570,23 +573,23 @@ importers: specifier: ^12.0.1 version: 12.0.1(commander@12.0.0) '@escape.tech/graphql-armor-max-aliases': - specifier: ^2.3.0 - version: 2.3.0 + specifier: ^2.4.0 + version: 2.4.0 '@escape.tech/graphql-armor-max-depth': - specifier: ^2.2.0 - version: 2.2.0 - '@escape.tech/graphql-armor-max-tokens': specifier: ^2.3.0 version: 2.3.0 + '@escape.tech/graphql-armor-max-tokens': + specifier: ^2.4.0 + version: 2.4.0 '@hono/node-server': - specifier: ^1.11.2 - version: 1.11.2 + specifier: ^1.11.3 + version: 1.11.3 '@ponder/utils': specifier: workspace:* version: link:../utils abitype: specifier: ^0.10.2 - version: 0.10.3(typescript@5.0.4) + version: 0.10.3 better-sqlite3: specifier: ^10.0.0 version: 10.0.0 @@ -615,14 +618,11 @@ importers: specifier: ^10.3.10 version: 10.3.10 graphql: - specifier: ^16.8.1 - version: 16.8.1 - graphql-type-json: - specifier: ^0.3.2 - version: 0.3.2(graphql@16.8.1) + specifier: ^16.8.2 + version: 16.8.2 graphql-yoga: - specifier: ^5.3.0 - version: 5.3.0(graphql@16.8.1) + specifier: ^5.3.1 + version: 5.3.1(graphql@16.8.2) http-terminator: specifier: ^3.2.0 version: 3.2.0 @@ -667,7 +667,7 @@ importers: version: 1.1.1(@types/node@20.11.24) vite-tsconfig-paths: specifier: ^4.3.1 - version: 4.3.1(typescript@5.0.4)(vite@5.0.10) + version: 4.3.1(vite@5.0.10) devDependencies: '@types/babel__code-frame': specifier: ^7.0.6 @@ -692,7 +692,7 @@ importers: version: 0.0.6 '@wagmi/cli': specifier: ^1.5.2 - version: 1.5.2(typescript@5.0.4) + version: 1.5.2 execa: specifier: ^8.0.1 version: 8.0.1 @@ -701,7 +701,7 @@ importers: version: 5.0.5 tsup: specifier: ^8.0.1 - version: 8.0.1(typescript@5.0.4) + version: 8.0.1 vitest: specifier: ^1.0.2 version: 1.0.2(@types/node@20.11.24) @@ -2399,40 +2399,21 @@ packages: '@jridgewell/trace-mapping': 0.3.9 dev: true - /@envelop/core@4.0.3: - resolution: {integrity: sha512-O0Vz8E0TObT6ijAob8jYFVJavcGywKThM3UAsxUIBBVPYZTMiqI9lo2gmAnbMUnrDcAYkUTZEW9FDYPRdF5l6g==} - engines: {node: '>=16.0.0'} - requiresBuild: true - dependencies: - '@envelop/types': 4.0.1 - tslib: 2.6.2 - dev: false - optional: true - /@envelop/core@5.0.1: resolution: {integrity: sha512-wxA8EyE1fPnlbP0nC/SFI7uU8wSNf4YjxZhAPu0P63QbgIvqHtHsH4L3/u+rsTruzhk3OvNRgQyLsMfaR9uzAQ==} engines: {node: '>=18.0.0'} requiresBuild: true dependencies: '@envelop/types': 5.0.0 - tslib: 2.6.2 + tslib: 2.6.3 dev: false - /@envelop/types@4.0.1: - resolution: {integrity: sha512-ULo27/doEsP7uUhm2iTnElx13qTO6I5FKvmLoX41cpfuw8x6e0NUFknoqhEsLzAbgz8xVS5mjwcxGCXh4lDYzg==} - engines: {node: '>=16.0.0'} - requiresBuild: true - dependencies: - tslib: 2.6.2 - dev: false - optional: true - /@envelop/types@5.0.0: resolution: {integrity: sha512-IPjmgSc4KpQRlO4qbEDnBEixvtb06WDmjKfi/7fkZaryh5HuOmTtixe1EupQI5XfXO8joc3d27uUZ0QdC++euA==} engines: {node: '>=18.0.0'} requiresBuild: true dependencies: - tslib: 2.6.2 + tslib: 2.6.3 dev: false /@esbuild/aix-ppc64@0.19.11: @@ -2817,41 +2798,41 @@ packages: requiresBuild: true optional: true - /@escape.tech/graphql-armor-max-aliases@2.3.0: - resolution: {integrity: sha512-h0AfPx929MWBnDlWnn/hcLHHNIAnUjws30OmyPLj9GqVmsBpj3338LELvORuuf3N1ciWI0xgkQd3NRSrmgr3ig==} - engines: {node: '>=16.0.0'} + /@escape.tech/graphql-armor-max-aliases@2.4.0: + resolution: {integrity: sha512-d4V9EgtPRG9HIoPHuanFNLHj1ENB1YkZi9FbiBiH88x5VahCjVpMXDgKQGkG6RUTOODU4XKp0/ZgaOq0pX5oEA==} + engines: {node: '>=18.0.0'} dependencies: - graphql: 16.8.1 + graphql: 16.8.2 optionalDependencies: '@envelop/core': 5.0.1 - '@escape.tech/graphql-armor-types': 0.5.0 + '@escape.tech/graphql-armor-types': 0.6.0 dev: false - /@escape.tech/graphql-armor-max-depth@2.2.0: - resolution: {integrity: sha512-v0z2yelQL614mYFpYL/iRkieq/7H2XKbvJ6RvbGMFFSqo3eSIz8fyX0f6pyswR7myQxki4ur0MFxSn8S5jjfqw==} - engines: {node: '>=16.0.0'} + /@escape.tech/graphql-armor-max-depth@2.3.0: + resolution: {integrity: sha512-EgqJU2yOaKaFeNDqMn18fIOI6UNjboWV950G9I39ebXyxsQmIaAx0Hs9hJoCBEHdLY9SCKWsEZFipHXqvaphdw==} + engines: {node: '>=18.0.0'} dependencies: - graphql: 16.8.1 + graphql: 16.8.2 optionalDependencies: - '@envelop/core': 4.0.3 - '@escape.tech/graphql-armor-types': 0.5.0 + '@envelop/core': 5.0.1 + '@escape.tech/graphql-armor-types': 0.6.0 dev: false - /@escape.tech/graphql-armor-max-tokens@2.3.0: - resolution: {integrity: sha512-4aqtUhT4ONUVWY6Z7crjPFyOK2/quUGHFU3G2+s4GYFxQHn3F5HjdI2KoY5ot2Sdijh4X+gx0ebBjUzriLNtbg==} - engines: {node: '>=16.0.0'} + /@escape.tech/graphql-armor-max-tokens@2.4.0: + resolution: {integrity: sha512-apKQBcYc6vsrITR+uKGXTC9yWV4zUEP4usb5zO0vebYT6e4KLcS2gwIQOsDLCnD5IyU5sUOzHBWmkFyBPz5keQ==} + engines: {node: '>=18.0.0'} dependencies: - graphql: 16.8.1 + graphql: 16.8.2 optionalDependencies: '@envelop/core': 5.0.1 - '@escape.tech/graphql-armor-types': 0.5.0 + '@escape.tech/graphql-armor-types': 0.6.0 dev: false - /@escape.tech/graphql-armor-types@0.5.0: - resolution: {integrity: sha512-a7KMhb1qVHFFWw4bvGYQI637YaIZRozbfc+Fj1Vv/pwnTCJOzOgnvKO8+WBXJsFFGJ2Kj+fRORmSpz7J+lJF1w==} + /@escape.tech/graphql-armor-types@0.6.0: + resolution: {integrity: sha512-Y3X6JgkB1N1MMaHNXaE2IeJWIs6wT4XcPvXM8PRWmT2DblZfY4NYiV1mh0GTInKdlnrEr5hquOR9XV+M3Da43w==} requiresBuild: true dependencies: - graphql: 16.8.1 + graphql: 16.8.2 dev: false optional: true @@ -3056,8 +3037,8 @@ packages: hasBin: true dependencies: '@rescript/std': 9.0.0 - graphql: 16.8.1 - graphql-import-node: 0.0.5(graphql@16.8.1) + graphql: 16.8.2 + graphql-import-node: 0.0.5(graphql@16.8.2) js-yaml: 4.1.0 dev: true @@ -3112,55 +3093,55 @@ packages: assemblyscript: 0.19.10 dev: true - /@graphql-tools/executor@1.2.6(graphql@16.8.1): + /@graphql-tools/executor@1.2.6(graphql@16.8.2): resolution: {integrity: sha512-+1kjfqzM5T2R+dCw7F4vdJ3CqG+fY/LYJyhNiWEFtq0ToLwYzR/KKyD8YuzTirEjSxWTVlcBh7endkx5n5F6ew==} engines: {node: '>=16.0.0'} peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - '@graphql-tools/utils': 10.2.0(graphql@16.8.1) - '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.1) - '@repeaterjs/repeater': 3.0.5 - graphql: 16.8.1 - tslib: 2.6.2 + '@graphql-tools/utils': 10.2.2(graphql@16.8.2) + '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.2) + '@repeaterjs/repeater': 3.0.6 + graphql: 16.8.2 + tslib: 2.6.3 value-or-promise: 1.0.12 dev: false - /@graphql-tools/merge@9.0.4(graphql@16.8.1): + /@graphql-tools/merge@9.0.4(graphql@16.8.2): resolution: {integrity: sha512-MivbDLUQ+4Q8G/Hp/9V72hbn810IJDEZQ57F01sHnlrrijyadibfVhaQfW/pNH+9T/l8ySZpaR/DpL5i+ruZ+g==} engines: {node: '>=16.0.0'} peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - '@graphql-tools/utils': 10.2.0(graphql@16.8.1) - graphql: 16.8.1 - tslib: 2.6.2 + '@graphql-tools/utils': 10.2.2(graphql@16.8.2) + graphql: 16.8.2 + tslib: 2.6.3 dev: false - /@graphql-tools/schema@10.0.3(graphql@16.8.1): - resolution: {integrity: sha512-p28Oh9EcOna6i0yLaCFOnkcBDQECVf3SCexT6ktb86QNj9idnkhI+tCxnwZDh58Qvjd2nURdkbevvoZkvxzCog==} + /@graphql-tools/schema@10.0.4(graphql@16.8.2): + resolution: {integrity: sha512-HuIwqbKxPaJujox25Ra4qwz0uQzlpsaBOzO6CVfzB/MemZdd+Gib8AIvfhQArK0YIN40aDran/yi+E5Xf0mQww==} engines: {node: '>=16.0.0'} peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - '@graphql-tools/merge': 9.0.4(graphql@16.8.1) - '@graphql-tools/utils': 10.2.0(graphql@16.8.1) - graphql: 16.8.1 - tslib: 2.6.2 + '@graphql-tools/merge': 9.0.4(graphql@16.8.2) + '@graphql-tools/utils': 10.2.2(graphql@16.8.2) + graphql: 16.8.2 + tslib: 2.6.3 value-or-promise: 1.0.12 dev: false - /@graphql-tools/utils@10.2.0(graphql@16.8.1): - resolution: {integrity: sha512-HYV7dO6pNA2nGKawygaBpk8y+vXOUjjzzO43W/Kb7EPRmXUEQKjHxPYRvQbiF72u1N3XxwGK5jnnFk9WVhUwYw==} + /@graphql-tools/utils@10.2.2(graphql@16.8.2): + resolution: {integrity: sha512-ueoplzHIgFfxhFrF4Mf/niU/tYHuO6Uekm2nCYU72qpI+7Hn9dA2/o5XOBvFXDk27Lp5VSvQY5WfmRbqwVxaYQ==} engines: {node: '>=16.0.0'} peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.1) + '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.2) cross-inspect: 1.0.0 dset: 3.1.3 - graphql: 16.8.1 - tslib: 2.6.2 + graphql: 16.8.2 + tslib: 2.6.3 dev: false /@graphql-typed-document-node/core@3.2.0(graphql@16.8.1): @@ -3168,14 +3149,22 @@ packages: peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - graphql: 16.8.1 + graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + dev: false + + /@graphql-typed-document-node/core@3.2.0(graphql@16.8.2): + resolution: {integrity: sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==} + peerDependencies: + graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + dependencies: + graphql: 16.8.2 dev: false /@graphql-yoga/logger@2.0.0: resolution: {integrity: sha512-Mg8psdkAp+YTG1OGmvU+xa6xpsAmSir0hhr3yFYPyLNwzUj95DdIwsMpKadDj9xDpYgJcH3Hp/4JMal9DhQimA==} engines: {node: '>=18.0.0'} dependencies: - tslib: 2.6.2 + tslib: 2.6.3 dev: false /@graphql-yoga/subscription@5.0.0: @@ -3183,17 +3172,17 @@ packages: engines: {node: '>=18.0.0'} dependencies: '@graphql-yoga/typed-event-target': 3.0.0 - '@repeaterjs/repeater': 3.0.5 + '@repeaterjs/repeater': 3.0.6 '@whatwg-node/events': 0.1.1 - tslib: 2.6.2 + tslib: 2.6.3 dev: false /@graphql-yoga/typed-event-target@3.0.0: resolution: {integrity: sha512-w+liuBySifrstuHbFrHoHAEyVnDFVib+073q8AeAJ/qqJfvFvAwUPLLtNohR/WDVRgSasfXtl3dcNuVJWN+rjg==} engines: {node: '>=18.0.0'} dependencies: - '@repeaterjs/repeater': 3.0.5 - tslib: 2.6.2 + '@repeaterjs/repeater': 3.0.6 + tslib: 2.6.3 dev: false /@headlessui/react@1.7.17(react-dom@18.2.0)(react@18.2.0): @@ -3208,8 +3197,8 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /@hono/node-server@1.11.2: - resolution: {integrity: sha512-JhX0nUC66GeDxpIdMKWDRMEwtQBa64CY907iAF1sYqb4m2p2PdSU7zkbnNhAZLg/6IjSlTuj6CF307JlBXVvpg==} + /@hono/node-server@1.11.3: + resolution: {integrity: sha512-mFg3qlKkDtMWSalX5Gyh6Zd3MXay0biGobFlyJ49i6R1smBBS1CYkNZbvwLlw+4sSrHO4ZiH7kj4TcLpl2Jr3g==} engines: {node: '>=18.14.1'} dev: false @@ -3845,8 +3834,8 @@ packages: react: 18.2.0 dev: false - /@repeaterjs/repeater@3.0.5: - resolution: {integrity: sha512-l3YHBLAol6d/IKnB9LhpD0cEZWAoe3eFKUyTYWmFmCO2Q/WOckxLQAUyMZWwZV2M/m3+4vgRoaolFqaII82/TA==} + /@repeaterjs/repeater@3.0.6: + resolution: {integrity: sha512-Javneu5lsuhwNCryN+pXH93VPQ8g0dBX7wItHFgYiwQmzE1sVdg5tWHiOgHywzL2W21XQopa7IwIEnNbmeUJYA==} dev: false /@rescript/std@9.0.0: @@ -4576,6 +4565,49 @@ packages: pretty-format: 29.7.0 dev: true + /@wagmi/cli@1.5.2: + resolution: {integrity: sha512-UfLMYhW6mQBCjR8A5s01Chf9GpHzdpcuuBuzJ36QGXcMSJAxylz5ImVZWfCRV0ct1UruydjKVSW1QSI6azNxRQ==} + engines: {node: '>=14'} + hasBin: true + peerDependencies: + '@wagmi/core': '>=1.0.0' + typescript: '>=5.0.4' + wagmi: '>=1.0.0' + peerDependenciesMeta: + '@wagmi/core': + optional: true + typescript: + optional: true + wagmi: + optional: true + dependencies: + abitype: 0.8.7(zod@3.22.4) + abort-controller: 3.0.0 + bundle-require: 3.1.2(esbuild@0.16.17) + cac: 6.7.14 + change-case: 4.1.2 + chokidar: 3.5.3 + dedent: 0.7.0 + detect-package-manager: 2.0.1 + dotenv: 16.3.1 + dotenv-expand: 10.0.0 + esbuild: 0.16.17 + execa: 6.1.0 + find-up: 6.3.0 + fs-extra: 10.1.0 + globby: 13.2.2 + node-fetch: 3.3.2 + ora: 6.3.1 + pathe: 1.1.1 + picocolors: 1.0.0 + prettier: 2.8.8 + viem: 1.21.4(zod@3.22.4) + zod: 3.22.4 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: true + /@wagmi/cli@1.5.2(typescript@5.0.4): resolution: {integrity: sha512-UfLMYhW6mQBCjR8A5s01Chf9GpHzdpcuuBuzJ36QGXcMSJAxylz5ImVZWfCRV0ct1UruydjKVSW1QSI6azNxRQ==} engines: {node: '>=14'} @@ -4639,8 +4671,8 @@ packages: web-streams-polyfill: 3.2.1 dev: true - /@whatwg-node/fetch@0.9.17: - resolution: {integrity: sha512-TDYP3CpCrxwxpiNY0UMNf096H5Ihf67BK1iKGegQl5u9SlpEDYrvnV71gWBGJm+Xm31qOy8ATgma9rm8Pe7/5Q==} + /@whatwg-node/fetch@0.9.18: + resolution: {integrity: sha512-hqoz6StCW+AjV/3N+vg0s1ah82ptdVUb9nH2ttj3UbySOXUvytWw2yqy8c1cKzyRk6mDD00G47qS3fZI9/gMjg==} engines: {node: '>=16.0.0'} dependencies: '@whatwg-node/node-fetch': 0.5.11 @@ -4665,15 +4697,15 @@ packages: '@whatwg-node/events': 0.1.1 busboy: 1.6.0 fast-querystring: 1.1.2 - tslib: 2.6.2 + tslib: 2.6.3 dev: false /@whatwg-node/server@0.9.34: resolution: {integrity: sha512-1sHRjqUtZIyTR2m2dS/dJpzS5OcNDpPuUSVDa2PoEgzYVKr4GsqJaYtRaEXXFohvvyh6PkouYCc1rE7jMDWVCA==} engines: {node: '>=16.0.0'} dependencies: - '@whatwg-node/fetch': 0.9.17 - tslib: 2.6.2 + '@whatwg-node/fetch': 0.9.18 + tslib: 2.6.3 dev: false /JSONStream@1.3.2: @@ -4692,6 +4724,18 @@ packages: through: 2.3.8 dev: true + /abitype@0.10.3: + resolution: {integrity: sha512-tRN+7XIa7J9xugdbRzFv/95ka5ivR/sRe01eiWvM0HWWjHuigSZEACgKa0sj4wGuekTDtghCx+5Izk/cOi78pQ==} + peerDependencies: + typescript: '>=5.0.4' + zod: ^3 >=3.22.0 + peerDependenciesMeta: + typescript: + optional: true + zod: + optional: true + dev: false + /abitype@0.10.3(typescript@5.0.4): resolution: {integrity: sha512-tRN+7XIa7J9xugdbRzFv/95ka5ivR/sRe01eiWvM0HWWjHuigSZEACgKa0sj4wGuekTDtghCx+5Izk/cOi78pQ==} peerDependencies: @@ -4704,6 +4748,7 @@ packages: optional: true dependencies: typescript: 5.0.4 + dev: true /abitype@0.10.3(typescript@5.3.3): resolution: {integrity: sha512-tRN+7XIa7J9xugdbRzFv/95ka5ivR/sRe01eiWvM0HWWjHuigSZEACgKa0sj4wGuekTDtghCx+5Izk/cOi78pQ==} @@ -4732,6 +4777,18 @@ packages: zod: 3.22.4 dev: true + /abitype@0.8.7(zod@3.22.4): + resolution: {integrity: sha512-wQ7hV8Yg/yKmGyFpqrNZufCxbszDe5es4AZGYPBitocfSqXtjrTG9JMWFcc4N30ukl2ve48aBTwt7NJxVQdU3w==} + peerDependencies: + typescript: '>=5.0.4' + zod: ^3 >=3.19.1 + peerDependenciesMeta: + zod: + optional: true + dependencies: + zod: 3.22.4 + dev: true + /abitype@0.9.8(typescript@5.0.4)(zod@3.22.4): resolution: {integrity: sha512-puLifILdm+8sjyss4S+fsUN09obiT1g2YW6CtcQF+QDzxR0euzgEB29MZujC6zMk2a6SVmtttq1fc6+YFA7WYQ==} peerDependencies: @@ -4760,6 +4817,20 @@ packages: dependencies: typescript: 5.3.3 + /abitype@0.9.8(zod@3.22.4): + resolution: {integrity: sha512-puLifILdm+8sjyss4S+fsUN09obiT1g2YW6CtcQF+QDzxR0euzgEB29MZujC6zMk2a6SVmtttq1fc6+YFA7WYQ==} + peerDependencies: + typescript: '>=5.0.4' + zod: ^3 >=3.19.1 + peerDependenciesMeta: + typescript: + optional: true + zod: + optional: true + dependencies: + zod: 3.22.4 + dev: true + /abitype@1.0.0(typescript@5.0.4): resolution: {integrity: sha512-NMeMah//6bJ56H5XRj8QCV4AwuW6hB6zqz2LnhhLdcWVQOsXki6/Pn3APeqxCma62nXIcmZWdu1DlHWS74umVQ==} peerDependencies: @@ -6069,7 +6140,7 @@ packages: resolution: {integrity: sha512-4PFfn4b5ZN6FMNGSZlyb7wUhuN8wvj8t/VQHZdM4JsDcruGJ8L2kf9zao98QIrBPFCpdk27qst/AGTl7pL3ypQ==} engines: {node: '>=16.0.0'} dependencies: - tslib: 2.6.2 + tslib: 2.6.3 dev: false /cross-spawn@5.1.0: @@ -6505,6 +6576,17 @@ packages: ms: 2.1.3 dev: true + /debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + /debug@4.3.4(supports-color@8.1.1): resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -7778,6 +7860,16 @@ packages: resolution: {integrity: sha512-Rwix9pBtC1Nuy5wysTmKy+UjbDJpIfg8eHjw0rjZ1mX4GNLz1Bmd16uDpI3Gk1i70Fgcs8Csg2lPm8HULFg9DQ==} dev: false + /follow-redirects@1.15.6: + resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: true + /follow-redirects@1.15.6(debug@4.3.4): resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} engines: {node: '>=4.0'} @@ -8189,12 +8281,12 @@ packages: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} dev: true - /graphql-import-node@0.0.5(graphql@16.8.1): + /graphql-import-node@0.0.5(graphql@16.8.2): resolution: {integrity: sha512-OXbou9fqh9/Lm7vwXT0XoRN9J5+WCYKnbiTalgFDvkQERITRmcfncZs6aVABedd5B85yQU5EULS4a5pnbpuI0Q==} peerDependencies: graphql: '*' dependencies: - graphql: 16.8.1 + graphql: 16.8.2 dev: true /graphql-request@6.1.0(graphql@16.8.1): @@ -8204,37 +8296,29 @@ packages: dependencies: '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.1) cross-fetch: 3.1.8 - graphql: 16.8.1 + graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) transitivePeerDependencies: - encoding dev: false - /graphql-type-json@0.3.2(graphql@16.8.1): - resolution: {integrity: sha512-J+vjof74oMlCWXSvt0DOf2APEdZOCdubEvGDUAlqH//VBYcOYsGgRW7Xzorr44LvkjiuvecWc8fChxuZZbChtg==} - peerDependencies: - graphql: '>=0.8.0' - dependencies: - graphql: 16.8.1 - dev: false - - /graphql-yoga@5.3.0(graphql@16.8.1): - resolution: {integrity: sha512-6mXoGE5AMN6YNugJvjBFIQ0dFUiOMloN0dAzLL8GFt4iJ5WlWRgjdzGHod2zZz7yWQokEVD42DHgrc7NY3Dm0w==} + /graphql-yoga@5.3.1(graphql@16.8.2): + resolution: {integrity: sha512-n918QV6TF7xTjb9ASnozgsr4ydMc08c+x4eRAWKxxWVwSnzdP2xeN2zw1ljIzRD0ccSCNoBajGDKwcZkJDitPA==} engines: {node: '>=18.0.0'} peerDependencies: graphql: ^15.2.0 || ^16.0.0 dependencies: '@envelop/core': 5.0.1 - '@graphql-tools/executor': 1.2.6(graphql@16.8.1) - '@graphql-tools/schema': 10.0.3(graphql@16.8.1) - '@graphql-tools/utils': 10.2.0(graphql@16.8.1) + '@graphql-tools/executor': 1.2.6(graphql@16.8.2) + '@graphql-tools/schema': 10.0.4(graphql@16.8.2) + '@graphql-tools/utils': 10.2.2(graphql@16.8.2) '@graphql-yoga/logger': 2.0.0 '@graphql-yoga/subscription': 5.0.0 - '@whatwg-node/fetch': 0.9.17 + '@whatwg-node/fetch': 0.9.18 '@whatwg-node/server': 0.9.34 dset: 3.1.3 - graphql: 16.8.1 - lru-cache: 10.1.0 - tslib: 2.6.2 + graphql: 16.8.2 + lru-cache: 10.2.2 + tslib: 2.6.3 dev: false /graphql@15.5.0: @@ -8242,9 +8326,15 @@ packages: engines: {node: '>= 10.x'} dev: true - /graphql@16.8.1: + /graphql@16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a): resolution: {integrity: sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} + dev: false + patched: true + + /graphql@16.8.2: + resolution: {integrity: sha512-cvVIBILwuoSyD54U4cF/UXDh5yAobhNV/tPygI4lZhgOIJQE/WLWC4waBRb4I6bDVYb3OVx3lfHbaQOEoUD5sg==} + engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} /gray-matter@4.0.3: resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} @@ -8417,6 +8507,24 @@ packages: zwitch: 2.0.4 dev: false + /hast-util-raw@9.0.4: + resolution: {integrity: sha512-LHE65TD2YiNsHD3YuXcKPHXPLuYh/gjp12mOfU8jxSrm1f/yJpsb0F/KKljS6U9LJoP0Ux+tCe8iJ2AsPzTdgA==} + dependencies: + '@types/hast': 3.0.4 + '@types/unist': 3.0.2 + '@ungap/structured-clone': 1.2.0 + hast-util-from-parse5: 8.0.1 + hast-util-to-parse5: 8.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.2.0 + parse5: 7.1.2 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + vfile: 6.0.1 + web-namespaces: 2.0.1 + zwitch: 2.0.4 + dev: false + /hast-util-to-estree@2.3.3: resolution: {integrity: sha512-ihhPIUPxN0v0w6M5+IiAZZrn0LH2uZomeWwhn7uP7avZC6TE7lIiEh2yBMPr5+zi1aUCXq6VoYRgs2Bw9xmycQ==} dependencies: @@ -8469,10 +8577,10 @@ packages: '@types/unist': 3.0.2 ccount: 2.0.1 comma-separated-tokens: 2.0.3 - hast-util-raw: 9.0.3 + hast-util-raw: 9.0.4 hast-util-whitespace: 3.0.0 html-void-elements: 3.0.0 - mdast-util-to-hast: 13.1.0 + mdast-util-to-hast: 13.2.0 property-information: 6.5.0 space-separated-tokens: 2.0.2 stringify-entities: 4.0.4 @@ -8572,7 +8680,7 @@ packages: engines: {node: '>=8.0.0'} dependencies: eventemitter3: 4.0.7 - follow-redirects: 1.15.6(debug@4.3.4) + follow-redirects: 1.15.6 requires-port: 1.0.0 transitivePeerDependencies: - debug @@ -9777,6 +9885,11 @@ packages: resolution: {integrity: sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==} engines: {node: 14 || >=16.14} + /lru-cache@10.2.2: + resolution: {integrity: sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==} + engines: {node: 14 || >=16.14} + dev: false + /lru-cache@4.1.5: resolution: {integrity: sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==} dependencies: @@ -10125,6 +10238,20 @@ packages: vfile: 6.0.1 dev: false + /mdast-util-to-hast@13.2.0: + resolution: {integrity: sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==} + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.2.0 + devlop: 1.1.0 + micromark-util-sanitize-uri: 2.0.0 + trim-lines: 3.0.1 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + vfile: 6.0.1 + dev: false + /mdast-util-to-markdown@1.5.0: resolution: {integrity: sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A==} dependencies: @@ -11865,6 +11992,22 @@ packages: camelcase-css: 2.0.1 postcss: 8.4.32 + /postcss-load-config@4.0.2: + resolution: {integrity: sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==} + engines: {node: '>= 14'} + peerDependencies: + postcss: '>=8.0.9' + ts-node: '>=9.0.0' + peerDependenciesMeta: + postcss: + optional: true + ts-node: + optional: true + dependencies: + lilconfig: 3.0.0 + yaml: 2.3.4 + dev: true + /postcss-load-config@4.0.2(postcss@8.4.32): resolution: {integrity: sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==} engines: {node: '>= 14'} @@ -13745,7 +13888,7 @@ packages: yn: 3.1.1 dev: true - /tsconfck@3.0.1(typescript@5.0.4): + /tsconfck@3.0.1: resolution: {integrity: sha512-7ppiBlF3UEddCLeI1JRx5m2Ryq+xk4JrZuq4EuYXykipebaq1dV0Fhgr1hb7CkmHt32QSgOZlcqVLEtHBG4/mg==} engines: {node: ^18 || >=20} hasBin: true @@ -13754,8 +13897,6 @@ packages: peerDependenciesMeta: typescript: optional: true - dependencies: - typescript: 5.0.4 dev: false /tsconfig-paths@3.15.0: @@ -13770,6 +13911,48 @@ packages: /tslib@2.6.2: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + /tslib@2.6.3: + resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==} + dev: false + + /tsup@8.0.1: + resolution: {integrity: sha512-hvW7gUSG96j53ZTSlT4j/KL0q1Q2l6TqGBFc6/mu/L46IoNWqLLUzLRLP1R8Q7xrJTmkDxxDoojV5uCVs1sVOg==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + '@microsoft/api-extractor': ^7.36.0 + '@swc/core': ^1 + postcss: ^8.4.12 + typescript: '>=4.5.0' + peerDependenciesMeta: + '@microsoft/api-extractor': + optional: true + '@swc/core': + optional: true + postcss: + optional: true + typescript: + optional: true + dependencies: + bundle-require: 4.0.2(esbuild@0.19.11) + cac: 6.7.14 + chokidar: 3.5.3 + debug: 4.3.4 + esbuild: 0.19.11 + execa: 5.1.1 + globby: 11.1.0 + joycon: 3.1.1 + postcss-load-config: 4.0.2 + resolve-from: 5.0.0 + rollup: 4.9.2 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tree-kill: 1.2.2 + transitivePeerDependencies: + - supports-color + - ts-node + dev: true + /tsup@8.0.1(typescript@5.0.4): resolution: {integrity: sha512-hvW7gUSG96j53ZTSlT4j/KL0q1Q2l6TqGBFc6/mu/L46IoNWqLLUzLRLP1R8Q7xrJTmkDxxDoojV5uCVs1sVOg==} engines: {node: '>=18'} @@ -14430,6 +14613,28 @@ packages: - zod dev: false + /viem@1.21.4(zod@3.22.4): + resolution: {integrity: sha512-BNVYdSaUjeS2zKQgPs+49e5JKocfo60Ib2yiXOWBT6LuVxY1I/6fFX3waEtpXvL1Xn4qu+BVitVtMh9lyThyhQ==} + peerDependencies: + typescript: '>=5.0.4' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + '@adraffy/ens-normalize': 1.10.0 + '@noble/curves': 1.2.0 + '@noble/hashes': 1.3.2 + '@scure/bip32': 1.3.2 + '@scure/bip39': 1.2.1 + abitype: 0.9.8(zod@3.22.4) + isows: 1.0.3(ws@8.13.0) + ws: 8.13.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + - zod + dev: true + /viem@2.7.20(typescript@5.0.4): resolution: {integrity: sha512-S31a24LWEjqXAjw1A+3/xALo+4eiYKklAjLtlLdPhA0cp+Kv6GcgruNVTktP8pKIGNYvpyQ+HA9PJyUhVXPdDw==} peerDependencies: @@ -14459,7 +14664,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 pathe: 1.1.1 picocolors: 1.0.0 vite: 5.0.10(@types/node@20.11.24) @@ -14480,7 +14685,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 pathe: 1.1.1 picocolors: 1.0.0 vite: 5.0.10(@types/node@20.11.24) @@ -14495,7 +14700,7 @@ packages: - terser dev: false - /vite-tsconfig-paths@4.3.1(typescript@5.0.4)(vite@5.0.10): + /vite-tsconfig-paths@4.3.1(vite@5.0.10): resolution: {integrity: sha512-cfgJwcGOsIxXOLU/nELPny2/LUD/lcf1IbfyeKTv2bsupVbTH/xpFtdQlBmIP1GEK2CjjLxYhFfB+QODFAx5aw==} peerDependencies: vite: '*' @@ -14503,9 +14708,9 @@ packages: vite: optional: true dependencies: - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 globrex: 0.1.2 - tsconfck: 3.0.1(typescript@5.0.4) + tsconfck: 3.0.1 vite: 5.0.10(@types/node@20.11.24) transitivePeerDependencies: - supports-color @@ -14581,7 +14786,7 @@ packages: acorn-walk: 8.3.1 cac: 6.7.14 chai: 4.3.10 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 execa: 8.0.1 local-pkg: 0.5.0 magic-string: 0.30.5 From 821fb29d194f10b176ab999fb6964a8473467283 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 16:55:15 -0400 Subject: [PATCH 017/122] remove patch --- package.json | 1 - patches/graphql@16.8.1.patch | 3 - pnpm-lock.yaml | 217 +++-------------------------------- 3 files changed, 18 insertions(+), 203 deletions(-) delete mode 100644 patches/graphql@16.8.1.patch diff --git a/package.json b/package.json index 29b9b15ea..26ce98d83 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,6 @@ "packageManager": "pnpm@8.6.10", "pnpm": { "patchedDependencies": { - "graphql@16.8.1": "patches/graphql@16.8.1.patch", "detect-package-manager@3.0.1": "patches/detect-package-manager@3.0.1.patch" }, "peerDependencyRules": { diff --git a/patches/graphql@16.8.1.patch b/patches/graphql@16.8.1.patch deleted file mode 100644 index 730b48159..000000000 --- a/patches/graphql@16.8.1.patch +++ /dev/null @@ -1,3 +0,0 @@ -diff --git a/index.mjs b/index.mjs -deleted file mode 100644 -index ba8672e675b69675b8d37d3aa521ee2298eacdcc..0000000000000000000000000000000000000000 \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b851e9506..2eac579f9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,9 +8,6 @@ patchedDependencies: detect-package-manager@3.0.1: hash: tkhlb7gk5ij4zxchvtwu3teirq path: patches/detect-package-manager@3.0.1.patch - graphql@16.8.1: - hash: 3zvcnrptpojleshpmtp6be677a - path: patches/graphql@16.8.1.patch importers: @@ -479,7 +476,7 @@ importers: version: 5.12.2(react@18.2.0) graphql: specifier: ^16.8.1 - version: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + version: 16.8.1 graphql-request: specifier: ^6.1.0 version: 6.1.0(graphql@16.8.1) @@ -589,7 +586,7 @@ importers: version: link:../utils abitype: specifier: ^0.10.2 - version: 0.10.3 + version: 0.10.3(typescript@5.0.4) better-sqlite3: specifier: ^10.0.0 version: 10.0.0 @@ -667,7 +664,7 @@ importers: version: 1.1.1(@types/node@20.11.24) vite-tsconfig-paths: specifier: ^4.3.1 - version: 4.3.1(vite@5.0.10) + version: 4.3.1(typescript@5.0.4)(vite@5.0.10) devDependencies: '@types/babel__code-frame': specifier: ^7.0.6 @@ -692,7 +689,7 @@ importers: version: 0.0.6 '@wagmi/cli': specifier: ^1.5.2 - version: 1.5.2 + version: 1.5.2(typescript@5.0.4) execa: specifier: ^8.0.1 version: 8.0.1 @@ -701,7 +698,7 @@ importers: version: 5.0.5 tsup: specifier: ^8.0.1 - version: 8.0.1 + version: 8.0.1(typescript@5.0.4) vitest: specifier: ^1.0.2 version: 1.0.2(@types/node@20.11.24) @@ -3149,7 +3146,7 @@ packages: peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 dependencies: - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 dev: false /@graphql-typed-document-node/core@3.2.0(graphql@16.8.2): @@ -4565,49 +4562,6 @@ packages: pretty-format: 29.7.0 dev: true - /@wagmi/cli@1.5.2: - resolution: {integrity: sha512-UfLMYhW6mQBCjR8A5s01Chf9GpHzdpcuuBuzJ36QGXcMSJAxylz5ImVZWfCRV0ct1UruydjKVSW1QSI6azNxRQ==} - engines: {node: '>=14'} - hasBin: true - peerDependencies: - '@wagmi/core': '>=1.0.0' - typescript: '>=5.0.4' - wagmi: '>=1.0.0' - peerDependenciesMeta: - '@wagmi/core': - optional: true - typescript: - optional: true - wagmi: - optional: true - dependencies: - abitype: 0.8.7(zod@3.22.4) - abort-controller: 3.0.0 - bundle-require: 3.1.2(esbuild@0.16.17) - cac: 6.7.14 - change-case: 4.1.2 - chokidar: 3.5.3 - dedent: 0.7.0 - detect-package-manager: 2.0.1 - dotenv: 16.3.1 - dotenv-expand: 10.0.0 - esbuild: 0.16.17 - execa: 6.1.0 - find-up: 6.3.0 - fs-extra: 10.1.0 - globby: 13.2.2 - node-fetch: 3.3.2 - ora: 6.3.1 - pathe: 1.1.1 - picocolors: 1.0.0 - prettier: 2.8.8 - viem: 1.21.4(zod@3.22.4) - zod: 3.22.4 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - dev: true - /@wagmi/cli@1.5.2(typescript@5.0.4): resolution: {integrity: sha512-UfLMYhW6mQBCjR8A5s01Chf9GpHzdpcuuBuzJ36QGXcMSJAxylz5ImVZWfCRV0ct1UruydjKVSW1QSI6azNxRQ==} engines: {node: '>=14'} @@ -4724,18 +4678,6 @@ packages: through: 2.3.8 dev: true - /abitype@0.10.3: - resolution: {integrity: sha512-tRN+7XIa7J9xugdbRzFv/95ka5ivR/sRe01eiWvM0HWWjHuigSZEACgKa0sj4wGuekTDtghCx+5Izk/cOi78pQ==} - peerDependencies: - typescript: '>=5.0.4' - zod: ^3 >=3.22.0 - peerDependenciesMeta: - typescript: - optional: true - zod: - optional: true - dev: false - /abitype@0.10.3(typescript@5.0.4): resolution: {integrity: sha512-tRN+7XIa7J9xugdbRzFv/95ka5ivR/sRe01eiWvM0HWWjHuigSZEACgKa0sj4wGuekTDtghCx+5Izk/cOi78pQ==} peerDependencies: @@ -4748,7 +4690,6 @@ packages: optional: true dependencies: typescript: 5.0.4 - dev: true /abitype@0.10.3(typescript@5.3.3): resolution: {integrity: sha512-tRN+7XIa7J9xugdbRzFv/95ka5ivR/sRe01eiWvM0HWWjHuigSZEACgKa0sj4wGuekTDtghCx+5Izk/cOi78pQ==} @@ -4777,18 +4718,6 @@ packages: zod: 3.22.4 dev: true - /abitype@0.8.7(zod@3.22.4): - resolution: {integrity: sha512-wQ7hV8Yg/yKmGyFpqrNZufCxbszDe5es4AZGYPBitocfSqXtjrTG9JMWFcc4N30ukl2ve48aBTwt7NJxVQdU3w==} - peerDependencies: - typescript: '>=5.0.4' - zod: ^3 >=3.19.1 - peerDependenciesMeta: - zod: - optional: true - dependencies: - zod: 3.22.4 - dev: true - /abitype@0.9.8(typescript@5.0.4)(zod@3.22.4): resolution: {integrity: sha512-puLifILdm+8sjyss4S+fsUN09obiT1g2YW6CtcQF+QDzxR0euzgEB29MZujC6zMk2a6SVmtttq1fc6+YFA7WYQ==} peerDependencies: @@ -4817,20 +4746,6 @@ packages: dependencies: typescript: 5.3.3 - /abitype@0.9.8(zod@3.22.4): - resolution: {integrity: sha512-puLifILdm+8sjyss4S+fsUN09obiT1g2YW6CtcQF+QDzxR0euzgEB29MZujC6zMk2a6SVmtttq1fc6+YFA7WYQ==} - peerDependencies: - typescript: '>=5.0.4' - zod: ^3 >=3.19.1 - peerDependenciesMeta: - typescript: - optional: true - zod: - optional: true - dependencies: - zod: 3.22.4 - dev: true - /abitype@1.0.0(typescript@5.0.4): resolution: {integrity: sha512-NMeMah//6bJ56H5XRj8QCV4AwuW6hB6zqz2LnhhLdcWVQOsXki6/Pn3APeqxCma62nXIcmZWdu1DlHWS74umVQ==} peerDependencies: @@ -6576,17 +6491,6 @@ packages: ms: 2.1.3 dev: true - /debug@4.3.4: - resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - dependencies: - ms: 2.1.2 - /debug@4.3.4(supports-color@8.1.1): resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -7860,16 +7764,6 @@ packages: resolution: {integrity: sha512-Rwix9pBtC1Nuy5wysTmKy+UjbDJpIfg8eHjw0rjZ1mX4GNLz1Bmd16uDpI3Gk1i70Fgcs8Csg2lPm8HULFg9DQ==} dev: false - /follow-redirects@1.15.6: - resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - dev: true - /follow-redirects@1.15.6(debug@4.3.4): resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} engines: {node: '>=4.0'} @@ -8296,7 +8190,7 @@ packages: dependencies: '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.1) cross-fetch: 3.1.8 - graphql: 16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a) + graphql: 16.8.1 transitivePeerDependencies: - encoding dev: false @@ -8326,11 +8220,10 @@ packages: engines: {node: '>= 10.x'} dev: true - /graphql@16.8.1(patch_hash=3zvcnrptpojleshpmtp6be677a): + /graphql@16.8.1: resolution: {integrity: sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} dev: false - patched: true /graphql@16.8.2: resolution: {integrity: sha512-cvVIBILwuoSyD54U4cF/UXDh5yAobhNV/tPygI4lZhgOIJQE/WLWC4waBRb4I6bDVYb3OVx3lfHbaQOEoUD5sg==} @@ -8680,7 +8573,7 @@ packages: engines: {node: '>=8.0.0'} dependencies: eventemitter3: 4.0.7 - follow-redirects: 1.15.6 + follow-redirects: 1.15.6(debug@4.3.4) requires-port: 1.0.0 transitivePeerDependencies: - debug @@ -11992,22 +11885,6 @@ packages: camelcase-css: 2.0.1 postcss: 8.4.32 - /postcss-load-config@4.0.2: - resolution: {integrity: sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==} - engines: {node: '>= 14'} - peerDependencies: - postcss: '>=8.0.9' - ts-node: '>=9.0.0' - peerDependenciesMeta: - postcss: - optional: true - ts-node: - optional: true - dependencies: - lilconfig: 3.0.0 - yaml: 2.3.4 - dev: true - /postcss-load-config@4.0.2(postcss@8.4.32): resolution: {integrity: sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==} engines: {node: '>= 14'} @@ -13888,7 +13765,7 @@ packages: yn: 3.1.1 dev: true - /tsconfck@3.0.1: + /tsconfck@3.0.1(typescript@5.0.4): resolution: {integrity: sha512-7ppiBlF3UEddCLeI1JRx5m2Ryq+xk4JrZuq4EuYXykipebaq1dV0Fhgr1hb7CkmHt32QSgOZlcqVLEtHBG4/mg==} engines: {node: ^18 || >=20} hasBin: true @@ -13897,6 +13774,8 @@ packages: peerDependenciesMeta: typescript: optional: true + dependencies: + typescript: 5.0.4 dev: false /tsconfig-paths@3.15.0: @@ -13915,44 +13794,6 @@ packages: resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==} dev: false - /tsup@8.0.1: - resolution: {integrity: sha512-hvW7gUSG96j53ZTSlT4j/KL0q1Q2l6TqGBFc6/mu/L46IoNWqLLUzLRLP1R8Q7xrJTmkDxxDoojV5uCVs1sVOg==} - engines: {node: '>=18'} - hasBin: true - peerDependencies: - '@microsoft/api-extractor': ^7.36.0 - '@swc/core': ^1 - postcss: ^8.4.12 - typescript: '>=4.5.0' - peerDependenciesMeta: - '@microsoft/api-extractor': - optional: true - '@swc/core': - optional: true - postcss: - optional: true - typescript: - optional: true - dependencies: - bundle-require: 4.0.2(esbuild@0.19.11) - cac: 6.7.14 - chokidar: 3.5.3 - debug: 4.3.4 - esbuild: 0.19.11 - execa: 5.1.1 - globby: 11.1.0 - joycon: 3.1.1 - postcss-load-config: 4.0.2 - resolve-from: 5.0.0 - rollup: 4.9.2 - source-map: 0.8.0-beta.0 - sucrase: 3.35.0 - tree-kill: 1.2.2 - transitivePeerDependencies: - - supports-color - - ts-node - dev: true - /tsup@8.0.1(typescript@5.0.4): resolution: {integrity: sha512-hvW7gUSG96j53ZTSlT4j/KL0q1Q2l6TqGBFc6/mu/L46IoNWqLLUzLRLP1R8Q7xrJTmkDxxDoojV5uCVs1sVOg==} engines: {node: '>=18'} @@ -14613,28 +14454,6 @@ packages: - zod dev: false - /viem@1.21.4(zod@3.22.4): - resolution: {integrity: sha512-BNVYdSaUjeS2zKQgPs+49e5JKocfo60Ib2yiXOWBT6LuVxY1I/6fFX3waEtpXvL1Xn4qu+BVitVtMh9lyThyhQ==} - peerDependencies: - typescript: '>=5.0.4' - peerDependenciesMeta: - typescript: - optional: true - dependencies: - '@adraffy/ens-normalize': 1.10.0 - '@noble/curves': 1.2.0 - '@noble/hashes': 1.3.2 - '@scure/bip32': 1.3.2 - '@scure/bip39': 1.2.1 - abitype: 0.9.8(zod@3.22.4) - isows: 1.0.3(ws@8.13.0) - ws: 8.13.0 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - zod - dev: true - /viem@2.7.20(typescript@5.0.4): resolution: {integrity: sha512-S31a24LWEjqXAjw1A+3/xALo+4eiYKklAjLtlLdPhA0cp+Kv6GcgruNVTktP8pKIGNYvpyQ+HA9PJyUhVXPdDw==} peerDependencies: @@ -14664,7 +14483,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) pathe: 1.1.1 picocolors: 1.0.0 vite: 5.0.10(@types/node@20.11.24) @@ -14685,7 +14504,7 @@ packages: hasBin: true dependencies: cac: 6.7.14 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) pathe: 1.1.1 picocolors: 1.0.0 vite: 5.0.10(@types/node@20.11.24) @@ -14700,7 +14519,7 @@ packages: - terser dev: false - /vite-tsconfig-paths@4.3.1(vite@5.0.10): + /vite-tsconfig-paths@4.3.1(typescript@5.0.4)(vite@5.0.10): resolution: {integrity: sha512-cfgJwcGOsIxXOLU/nELPny2/LUD/lcf1IbfyeKTv2bsupVbTH/xpFtdQlBmIP1GEK2CjjLxYhFfB+QODFAx5aw==} peerDependencies: vite: '*' @@ -14708,9 +14527,9 @@ packages: vite: optional: true dependencies: - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) globrex: 0.1.2 - tsconfck: 3.0.1 + tsconfck: 3.0.1(typescript@5.0.4) vite: 5.0.10(@types/node@20.11.24) transitivePeerDependencies: - supports-color @@ -14786,7 +14605,7 @@ packages: acorn-walk: 8.3.1 cac: 6.7.14 chai: 4.3.10 - debug: 4.3.4 + debug: 4.3.4(supports-color@8.1.1) execa: 8.0.1 local-pkg: 0.5.0 magic-string: 0.30.5 From f67fdb5e70c1a6cb8216315a28a45a51721522b5 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 17:38:58 -0400 Subject: [PATCH 018/122] direct sql --- examples/reference-erc20/ponder.server.ts | 12 ++++++++---- packages/core/src/bin/utils/run.ts | 4 ++++ packages/core/src/server/service.ts | 6 ++++++ packages/core/src/types/virtual.ts | 6 ++++++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/examples/reference-erc20/ponder.server.ts b/examples/reference-erc20/ponder.server.ts index d1c28ea6c..5208211d0 100644 --- a/examples/reference-erc20/ponder.server.ts +++ b/examples/reference-erc20/ponder.server.ts @@ -1,17 +1,21 @@ import { hono } from "@/generated"; import { graphQLMiddleware } from "@ponder/core"; +import { hexToBytes, zeroAddress } from "viem"; hono.use("/graphql", graphQLMiddleware()); hono.get("/router", async (c) => { const db = c.get("db"); - const account = await db.Account.findUnique({ - id: "0x0000000000000000000000000000000000000000", - }); + + const account = await db.query<{ balance: string }>( + `SELECT * FROM "Account" WHERE id = ${Buffer.from( + hexToBytes(zeroAddress), + )}`, + ); if (account === null) { return c.text("Not Found!"); } else { - return c.text(`Balance: ${account.balance.toString()}`); + return c.text(`Balance: ${account.rows[0]?.balance.toString() ?? "0"}`); } }); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index e18e2a660..a8d1fbe27 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -23,6 +23,7 @@ import { } from "@/utils/checkpoint.js"; import { never } from "@/utils/never.js"; import { createQueue } from "@ponder/common"; +import { sql } from "kysely"; export type RealtimeEvent = | { @@ -105,6 +106,9 @@ export async function run({ app: build.app, readonlyStore, schema, + query: (...query: any) => { + return sql.raw(query).execute(database.indexingDb); + }, common, }); diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index c42e75845..702f9f8dd 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -11,16 +11,19 @@ import { Hono } from "hono"; import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; +import { type QueryResult } from "kysely"; export async function createServer({ app, schema, readonlyStore, + query, common, }: { app?: Hono; schema: Schema; readonlyStore: ReadonlyStore; + query: (query: string) => Promise>; common: Common; }) { // Create hono app @@ -129,6 +132,9 @@ export async function createServer({ return acc; }, {}); + // @ts-ignore + db.query = query; + const contextMiddleware = createMiddleware(async (c, next) => { c.set("db", db); c.set("readonlyStore", readonlyStore); diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index 395206445..73d92f5ea 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -17,6 +17,7 @@ import type { TransactionReceipt, } from "@/types/eth.js"; import type { DatabaseModel } from "@/types/model.js"; +import type { QueryResult } from "kysely"; import type { Prettify } from "./utils.js"; export namespace Virtual { @@ -167,6 +168,11 @@ export namespace Virtual { "findUnique" | "findMany" > >; + } & { + query: ( + sqlFragment: string, + ...parameters: unknown[] + ) => Promise>; }; export type Context< From ef36400f1c17b23376828bec707a555f5ea16a03 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 17:58:35 -0400 Subject: [PATCH 019/122] search path --- examples/reference-erc20/ponder.server.ts | 11 ++++------- packages/core/src/bin/utils/run.ts | 4 +++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/examples/reference-erc20/ponder.server.ts b/examples/reference-erc20/ponder.server.ts index 5208211d0..05b5f12d0 100644 --- a/examples/reference-erc20/ponder.server.ts +++ b/examples/reference-erc20/ponder.server.ts @@ -1,21 +1,18 @@ import { hono } from "@/generated"; import { graphQLMiddleware } from "@ponder/core"; -import { hexToBytes, zeroAddress } from "viem"; hono.use("/graphql", graphQLMiddleware()); hono.get("/router", async (c) => { const db = c.get("db"); - const account = await db.query<{ balance: string }>( - `SELECT * FROM "Account" WHERE id = ${Buffer.from( - hexToBytes(zeroAddress), - )}`, + const account = await db.query<{ balance: bigint }>( + `SELECT * FROM "Account" LIMIT 1`, ); - if (account === null) { + if (account.rows.length === 0) { return c.text("Not Found!"); } else { - return c.text(`Balance: ${account.rows[0]?.balance.toString() ?? "0"}`); + return c.text(`Balance: ${account.rows[0]!.balance.toString()}`); } }); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index a8d1fbe27..b6b46cc9b 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -107,7 +107,9 @@ export async function run({ readonlyStore, schema, query: (...query: any) => { - return sql.raw(query).execute(database.indexingDb); + return sql + .raw(query) + .execute(database.indexingDb.withSchema(namespaceInfo.userNamespace)); }, common, }); From 5f36800c6e2003cff4906171dc971656885bc0a5 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 19:55:04 -0400 Subject: [PATCH 020/122] direct sql --- examples/reference-erc20/package.json | 2 +- examples/reference-erc20/ponder-env.d.ts | 4 +-- examples/reference-erc20/ponder.server.ts | 2 ++ package.json | 6 +--- packages/core/package.json | 14 ++++---- packages/core/src/common/codegen.ts | 4 +-- packages/core/src/server/service.ts | 18 +++++++--- packages/core/src/types/virtual.ts | 35 ++++++++++--------- pnpm-lock.yaml | 41 +++++++++-------------- 9 files changed, 61 insertions(+), 65 deletions(-) diff --git a/examples/reference-erc20/package.json b/examples/reference-erc20/package.json index d0bc6a886..8e893e9f6 100644 --- a/examples/reference-erc20/package.json +++ b/examples/reference-erc20/package.json @@ -12,7 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", - "hono": "^4.1.3", + "hono": "^4.4.6", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index f61470bb3..ec6a0c5aa 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -12,9 +12,7 @@ declare module "@/generated" { export const ponder: Virtual.Registry; - export const hono: Hono<{ - Variables: { db: Virtual.ReadonlyDb }; - }>; + export const hono: Virtual.Hono; export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< diff --git a/examples/reference-erc20/ponder.server.ts b/examples/reference-erc20/ponder.server.ts index 05b5f12d0..b099f9742 100644 --- a/examples/reference-erc20/ponder.server.ts +++ b/examples/reference-erc20/ponder.server.ts @@ -16,3 +16,5 @@ hono.get("/router", async (c) => { return c.text(`Balance: ${account.rows[0]!.balance.toString()}`); } }); + +hono.get("/_ponder/metrics", (c) => c.text("this is bad")); diff --git a/package.json b/package.json index 865933850..9ae2044c1 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,7 @@ "lint-staged": "^15.1.0", "simple-git-hooks": "^2.9.0", "typescript": "5.0.4", - "viem": "1.16.0", - "hono": "4.0.0" + "viem": "1.16.0" }, "lint-staged": { "*.ts": ["biome format --no-errors-on-unmatched", "biome check"], @@ -32,9 +31,6 @@ }, "packageManager": "pnpm@8.6.10", "pnpm": { - "patchedDependencies": { - "detect-package-manager@3.0.1": "patches/detect-package-manager@3.0.1.patch" - }, "peerDependencyRules": { "ignoreMissing": ["node-fetch"] } diff --git a/packages/core/package.json b/packages/core/package.json index 3c5cf7360..3a13e02ce 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -33,7 +33,6 @@ "typecheck": "tsc --noEmit" }, "peerDependencies": { - "hono": ">=4", "typescript": ">=5.0.4", "viem": ">=1.16" }, @@ -45,10 +44,10 @@ "dependencies": { "@babel/code-frame": "^7.23.4", "@commander-js/extra-typings": "^12.0.1", - "@escape.tech/graphql-armor-max-aliases": "^2.4.0", - "@escape.tech/graphql-armor-max-depth": "^2.3.0", - "@escape.tech/graphql-armor-max-tokens": "^2.4.0", - "@hono/node-server": "^1.11.3", + "@escape.tech/graphql-armor-max-aliases": "^2.3.0", + "@escape.tech/graphql-armor-max-depth": "^2.2.0", + "@escape.tech/graphql-armor-max-tokens": "^2.3.0", + "@hono/node-server": "^1.11.2", "@ponder/utils": "workspace:*", "abitype": "^0.10.2", "better-sqlite3": "^10.0.0", @@ -60,8 +59,9 @@ "emittery": "^1.0.1", "ethereum-bloom-filters": "^1.0.10", "glob": "^10.3.10", - "graphql": "^16.8.2", - "graphql-yoga": "^5.3.1", + "graphql": "^16.8.1", + "graphql-yoga": "^5.3.0", + "hono": "^4.4.2", "http-terminator": "^3.2.0", "ink": "^4.4.1", "kysely": "^0.26.3", diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index 8ae0af73e..e16b0ac56 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -17,9 +17,7 @@ declare module "@/generated" { export const ponder: Virtual.Registry; - export const hono: Hono<{ - Variables: { db: Virtual.ReadonlyDb }; - }>; + export const hono: Virtual.Hono; export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 5a7dfe83c..ae135524b 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -13,6 +13,13 @@ import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; import type { QueryResult } from "kysely"; +type Server = { + hono: Hono; + port: number; + setHealthy: () => void; + kill: () => Promise; +}; + export async function createServer({ app, schema, @@ -25,7 +32,7 @@ export async function createServer({ readonlyStore: ReadonlyStore; query: (query: string) => Promise>; common: Common; -}) { +}): Promise { // Create hono app const startTime = Date.now(); @@ -38,7 +45,7 @@ export async function createServer({ const metrics = await common.metrics.getMetrics(); return c.text(metrics); } catch (error) { - return c.json(error, 500); + return c.json(error as Error, 500); } }) .get("/health", async (c) => { @@ -145,8 +152,11 @@ export async function createServer({ const hono = new Hono() .use(metricsMiddleware) .route("/_ponder", ponderApp) - .use(contextMiddleware) - .route("/", app); + .use(contextMiddleware); + + if (app !== undefined) { + hono.route("/", app); + } // Create nodejs server diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index c6317f7d2..0b562dd14 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -17,6 +17,7 @@ import type { TransactionReceipt, } from "@/types/eth.js"; import type { DatabaseModel } from "@/types/model.js"; +import type { Hono as _Hono } from "hono"; import type { QueryResult } from "kysely"; import type { Prettify } from "./utils.js"; @@ -156,22 +157,24 @@ export namespace Virtual { >[property], > = ([base] extends [never] ? undefined : base) | override; - export type ReadonlyDb = { - [key in keyof InferSchemaType]: Prettify< - Pick< - DatabaseModel< - // @ts-ignore - InferSchemaType[key] - >, - "findUnique" | "findMany" - > - >; - } & { - query: ( - sqlFragment: string, - ...parameters: unknown[] - ) => Promise>; - }; + export type Hono = _Hono<{ + Variables: { + [key in keyof InferSchemaType]: Prettify< + Pick< + DatabaseModel< + // @ts-ignore + InferSchemaType[key] + >, + "findUnique" | "findMany" + > + >; + } & { + query: ( + sqlFragment: string, + ...parameters: unknown[] + ) => Promise>; + }; + }>; export type Context< config extends Config, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 682a104e8..949797d3d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,11 +4,6 @@ settings: autoInstallPeers: false excludeLinksFromLockfile: false -patchedDependencies: - detect-package-manager@3.0.1: - hash: tkhlb7gk5ij4zxchvtwu3teirq - path: patches/detect-package-manager@3.0.1.patch - importers: .: @@ -22,9 +17,6 @@ importers: '@changesets/cli': specifier: ^2.26.2 version: 2.27.1 - hono: - specifier: 4.0.0 - version: 4.0.0 lint-staged: specifier: ^15.1.0 version: 15.2.0 @@ -370,8 +362,8 @@ importers: specifier: workspace:* version: link:../../packages/core hono: - specifier: ^4.1.3 - version: 4.4.2 + specifier: ^4.4.6 + version: 4.4.6 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3) @@ -570,16 +562,16 @@ importers: specifier: ^12.0.1 version: 12.0.1(commander@12.0.0) '@escape.tech/graphql-armor-max-aliases': - specifier: ^2.4.0 + specifier: ^2.3.0 version: 2.4.0 '@escape.tech/graphql-armor-max-depth': - specifier: ^2.3.0 + specifier: ^2.2.0 version: 2.3.0 '@escape.tech/graphql-armor-max-tokens': - specifier: ^2.4.0 + specifier: ^2.3.0 version: 2.4.0 '@hono/node-server': - specifier: ^1.11.3 + specifier: ^1.11.2 version: 1.11.3 '@ponder/utils': specifier: workspace:* @@ -601,7 +593,7 @@ importers: version: 2.2.2 detect-package-manager: specifier: ^3.0.1 - version: 3.0.1(patch_hash=tkhlb7gk5ij4zxchvtwu3teirq) + version: 3.0.1 dotenv: specifier: ^16.3.1 version: 16.3.1 @@ -615,11 +607,14 @@ importers: specifier: ^10.3.10 version: 10.3.10 graphql: - specifier: ^16.8.2 + specifier: ^16.8.1 version: 16.8.2 graphql-yoga: - specifier: ^5.3.1 + specifier: ^5.3.0 version: 5.3.1(graphql@16.8.2) + hono: + specifier: ^4.4.2 + version: 4.4.6 http-terminator: specifier: ^3.2.0 version: 3.2.0 @@ -6631,13 +6626,12 @@ packages: execa: 5.1.1 dev: true - /detect-package-manager@3.0.1(patch_hash=tkhlb7gk5ij4zxchvtwu3teirq): + /detect-package-manager@3.0.1: resolution: {integrity: sha512-qoHDH6+lMcpJPAScE7+5CYj91W0mxZNXTwZPrCqi1KMk+x+AoQScQ2V1QyqTln1rHU5Haq5fikvOGHv+leKD8A==} engines: {node: '>=12'} dependencies: execa: 5.1.1 dev: false - patched: true /devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} @@ -8540,13 +8534,8 @@ packages: minimalistic-crypto-utils: 1.0.1 dev: true - /hono@4.0.0: - resolution: {integrity: sha512-8dKhuBBpRZEodUttQhrSFJ6PQqHRjXHyeeegfxOf132pvgbf0tOb9qqb7q7eYwAWpOcYrsUOsWdJ0sQIIovhZg==} - engines: {node: '>=16.0.0'} - dev: true - - /hono@4.4.2: - resolution: {integrity: sha512-bRhZ+BM9r04lRN2i9wiZ18yQNbZxHsmmRIItoAb43nRkHnIDsFhFh4mJ0seEs06FvenibpAgSVNHQ8ZzcDQx2A==} + /hono@4.4.6: + resolution: {integrity: sha512-XGRnoH8WONv60+PPvP9Sn067A9r/8JdHDJ5bgon0DVEHeR1cJPkWjv2aT+DBfMH9/mEkYa1+VEVFp1DT1lIwjw==} engines: {node: '>=16.0.0'} dev: false From f0468f8e28cb3cbfb94b01581e3c890bdc8b2a15 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 20:06:58 -0400 Subject: [PATCH 021/122] fix package.json --- examples/reference-erc20/ponder-env.d.ts | 3 +- examples/reference-erc20/ponder.server.ts | 2 -- package.json | 3 ++ packages/core/src/common/codegen.ts | 3 +- packages/core/src/types/virtual.ts | 34 ++++++++++++----------- pnpm-lock.yaml | 10 +++++-- 6 files changed, 31 insertions(+), 24 deletions(-) diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index ec6a0c5aa..ebd383c1d 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -5,14 +5,13 @@ declare module "@/generated" { import type { Virtual } from "@ponder/core"; - import type { Hono, Context as HonoContext } from "hono"; type config = typeof import("./ponder.config.ts").default; type schema = typeof import("./ponder.schema.ts").default; export const ponder: Virtual.Registry; - export const hono: Virtual.Hono; + export const hono: Virtual.App; export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< diff --git a/examples/reference-erc20/ponder.server.ts b/examples/reference-erc20/ponder.server.ts index b099f9742..05b5f12d0 100644 --- a/examples/reference-erc20/ponder.server.ts +++ b/examples/reference-erc20/ponder.server.ts @@ -16,5 +16,3 @@ hono.get("/router", async (c) => { return c.text(`Balance: ${account.rows[0]!.balance.toString()}`); } }); - -hono.get("/_ponder/metrics", (c) => c.text("this is bad")); diff --git a/package.json b/package.json index 9ae2044c1..ddf0782e4 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,9 @@ }, "packageManager": "pnpm@8.6.10", "pnpm": { + "patchedDependencies": { + "detect-package-manager@3.0.1": "patches/detect-package-manager@3.0.1.patch" + }, "peerDependencyRules": { "ignoreMissing": ["node-fetch"] } diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index e16b0ac56..13d0d27e8 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -10,14 +10,13 @@ export const ponderEnv = `// This file enables type checking and editor autocomp declare module "@/generated" { import type { Virtual } from "@ponder/core"; - import type { Hono, Context as HonoContext } from "hono"; type config = typeof import("./ponder.config.ts").default; type schema = typeof import("./ponder.schema.ts").default; export const ponder: Virtual.Registry; - export const hono: Virtual.Hono; + export const hono: Virtual.App; export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index 0b562dd14..438e585ae 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -17,7 +17,7 @@ import type { TransactionReceipt, } from "@/types/eth.js"; import type { DatabaseModel } from "@/types/model.js"; -import type { Hono as _Hono } from "hono"; +import type { Hono } from "hono"; import type { QueryResult } from "kysely"; import type { Prettify } from "./utils.js"; @@ -157,22 +157,24 @@ export namespace Virtual { >[property], > = ([base] extends [never] ? undefined : base) | override; - export type Hono = _Hono<{ + export type App = Hono<{ Variables: { - [key in keyof InferSchemaType]: Prettify< - Pick< - DatabaseModel< - // @ts-ignore - InferSchemaType[key] - >, - "findUnique" | "findMany" - > - >; - } & { - query: ( - sqlFragment: string, - ...parameters: unknown[] - ) => Promise>; + db: { + [key in keyof InferSchemaType]: Prettify< + Pick< + DatabaseModel< + // @ts-ignore + InferSchemaType[key] + >, + "findUnique" | "findMany" + > + >; + } & { + query: ( + sqlFragment: string, + ...parameters: unknown[] + ) => Promise>; + }; }; }>; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 949797d3d..6df82078a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,11 @@ settings: autoInstallPeers: false excludeLinksFromLockfile: false +patchedDependencies: + detect-package-manager@3.0.1: + hash: tkhlb7gk5ij4zxchvtwu3teirq + path: patches/detect-package-manager@3.0.1.patch + importers: .: @@ -593,7 +598,7 @@ importers: version: 2.2.2 detect-package-manager: specifier: ^3.0.1 - version: 3.0.1 + version: 3.0.1(patch_hash=tkhlb7gk5ij4zxchvtwu3teirq) dotenv: specifier: ^16.3.1 version: 16.3.1 @@ -6626,12 +6631,13 @@ packages: execa: 5.1.1 dev: true - /detect-package-manager@3.0.1: + /detect-package-manager@3.0.1(patch_hash=tkhlb7gk5ij4zxchvtwu3teirq): resolution: {integrity: sha512-qoHDH6+lMcpJPAScE7+5CYj91W0mxZNXTwZPrCqi1KMk+x+AoQScQ2V1QyqTln1rHU5Haq5fikvOGHv+leKD8A==} engines: {node: '>=12'} dependencies: execa: 5.1.1 dev: false + patched: true /devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} From 0134f38b1062e0374cf9333609d3ad83ccee1e40 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 17 Jun 2024 20:08:45 -0400 Subject: [PATCH 022/122] pnpm dedupe --- pnpm-lock.yaml | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6df82078a..bb8fc1577 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -440,7 +440,7 @@ importers: dependencies: forge-std: specifier: github:foundry-rs/forge-std - version: github.com/foundry-rs/forge-std/52715a217dc51d0de15877878ab8213f6cbbbab5 + version: github.com/foundry-rs/forge-std/f4a0353eaabd14128fe86f4a2b9a07b827f76e46 examples/with-foundry/ponder: dependencies: @@ -473,10 +473,10 @@ importers: version: 5.12.2(react@18.2.0) graphql: specifier: ^16.8.1 - version: 16.8.1 + version: 16.8.2 graphql-request: specifier: ^6.1.0 - version: 6.1.0(graphql@16.8.1) + version: 6.1.0(graphql@16.8.2) next: specifier: 14.0.3 version: 14.0.3(react-dom@18.2.0)(react@18.2.0) @@ -3161,14 +3161,6 @@ packages: tslib: 2.6.2 dev: false - /@graphql-typed-document-node/core@3.2.0(graphql@16.8.1): - resolution: {integrity: sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==} - peerDependencies: - graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - dependencies: - graphql: 16.8.1 - dev: false - /@graphql-typed-document-node/core@3.2.0(graphql@16.8.2): resolution: {integrity: sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==} peerDependencies: @@ -3222,6 +3214,7 @@ packages: /@humanwhocodes/config-array@0.11.13: resolution: {integrity: sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==} engines: {node: '>=10.10.0'} + deprecated: Use @eslint/config-array instead dependencies: '@humanwhocodes/object-schema': 2.0.1 debug: 4.3.4(supports-color@8.1.1) @@ -3237,6 +3230,7 @@ packages: /@humanwhocodes/object-schema@2.0.1: resolution: {integrity: sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==} + deprecated: Use @eslint/object-schema instead dev: true /@ipld/dag-cbor@7.0.3: @@ -8201,14 +8195,14 @@ packages: graphql: 16.8.2 dev: true - /graphql-request@6.1.0(graphql@16.8.1): + /graphql-request@6.1.0(graphql@16.8.2): resolution: {integrity: sha512-p+XPfS4q7aIpKVcgmnZKhMNqhltk20hfXtkaIkTfjjmiKMJ5xrt5c743cL03y/K7y1rg3WrIC49xGiEQ4mxdNw==} peerDependencies: graphql: 14 - 16 dependencies: - '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.1) + '@graphql-typed-document-node/core': 3.2.0(graphql@16.8.2) cross-fetch: 3.1.8 - graphql: 16.8.1 + graphql: 16.8.2 transitivePeerDependencies: - encoding dev: false @@ -8238,11 +8232,6 @@ packages: engines: {node: '>= 10.x'} dev: true - /graphql@16.8.1: - resolution: {integrity: sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==} - engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} - dev: false - /graphql@16.8.2: resolution: {integrity: sha512-cvVIBILwuoSyD54U4cF/UXDh5yAobhNV/tPygI4lZhgOIJQE/WLWC4waBRb4I6bDVYb3OVx3lfHbaQOEoUD5sg==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} @@ -14976,8 +14965,8 @@ packages: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} dev: false - github.com/foundry-rs/forge-std/52715a217dc51d0de15877878ab8213f6cbbbab5: - resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/52715a217dc51d0de15877878ab8213f6cbbbab5} + github.com/foundry-rs/forge-std/f4a0353eaabd14128fe86f4a2b9a07b827f76e46: + resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/f4a0353eaabd14128fe86f4a2b9a07b827f76e46} name: forge-std version: 1.8.2 dev: false From ea5e16de2981894611d7aedad9d605ca65df7ec2 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 18 Jun 2024 11:28:38 -0400 Subject: [PATCH 023/122] readonly db --- examples/reference-erc20/ponder.server.ts | 2 ++ packages/core/src/bin/utils/run.ts | 2 +- packages/core/src/database/postgres/service.ts | 11 ++++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/examples/reference-erc20/ponder.server.ts b/examples/reference-erc20/ponder.server.ts index 05b5f12d0..45f18d7aa 100644 --- a/examples/reference-erc20/ponder.server.ts +++ b/examples/reference-erc20/ponder.server.ts @@ -6,6 +6,8 @@ hono.use("/graphql", graphQLMiddleware()); hono.get("/router", async (c) => { const db = c.get("db"); + // await db.query(`UPDATE "Account" SET "isOwner" = 1`); + const account = await db.query<{ balance: bigint }>( `SELECT * FROM "Account" LIMIT 1`, ); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index b6b46cc9b..3f79e6ece 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -109,7 +109,7 @@ export async function run({ query: (...query: any) => { return sql .raw(query) - .execute(database.indexingDb.withSchema(namespaceInfo.userNamespace)); + .execute(database.readonlyDb.withSchema(namespaceInfo.userNamespace)); }, common, }); diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index cdf14a09e..6f8c749fa 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -151,7 +151,16 @@ export class PostgresDatabaseService implements BaseDatabaseService { this.readonlyDb = new HeadlessKysely({ name: "readonly", common, - dialect: new PostgresDialect({ pool: this.readonlyPool }), + dialect: new PostgresDialect({ + pool: this.readonlyPool, + onCreateConnection: async (connection) => { + await connection.executeQuery( + sql + .raw("SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY") + .compile(this.readonlyDb), + ); + }, + }), log(event) { if (event.level === "query") { common.metrics.ponder_postgres_query_total.inc({ pool: "readonly" }); From e3ed3b8cfb177519975d720366e6bd25cd1741e0 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 18 Jun 2024 11:52:56 -0400 Subject: [PATCH 024/122] directories --- .../src/{ => indexing}/index.ts | 0 .../{ponder.server.ts => src/server/index.ts} | 4 +- packages/core/src/bin/commands/serve.ts | 6 +- packages/core/src/bin/utils/run.ts | 2 +- packages/core/src/build/service.ts | 96 ++++++++++++------- packages/core/src/common/options.ts | 8 +- .../{middleware.test.ts => index.test.ts} | 14 +-- .../src/graphql/{middleware.ts => index.ts} | 2 +- packages/core/src/index.ts | 2 +- packages/core/src/indexing/addStackTrace.ts | 4 +- packages/core/src/server/service.ts | 12 ++- 11 files changed, 90 insertions(+), 60 deletions(-) rename examples/reference-erc20/src/{ => indexing}/index.ts (100%) rename examples/reference-erc20/{ponder.server.ts => src/server/index.ts} (81%) rename packages/core/src/graphql/{middleware.test.ts => index.test.ts} (94%) rename packages/core/src/graphql/{middleware.ts => index.ts} (98%) diff --git a/examples/reference-erc20/src/index.ts b/examples/reference-erc20/src/indexing/index.ts similarity index 100% rename from examples/reference-erc20/src/index.ts rename to examples/reference-erc20/src/indexing/index.ts diff --git a/examples/reference-erc20/ponder.server.ts b/examples/reference-erc20/src/server/index.ts similarity index 81% rename from examples/reference-erc20/ponder.server.ts rename to examples/reference-erc20/src/server/index.ts index 45f18d7aa..2b589d8ec 100644 --- a/examples/reference-erc20/ponder.server.ts +++ b/examples/reference-erc20/src/server/index.ts @@ -1,7 +1,7 @@ import { hono } from "@/generated"; -import { graphQLMiddleware } from "@ponder/core"; +import { graphql } from "@ponder/core"; -hono.use("/graphql", graphQLMiddleware()); +hono.use("/graphql", graphql()); hono.get("/router", async (c) => { const db = c.get("db"); diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index 43da08b9e..a74acd4a4 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -66,7 +66,7 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { properties: { cli_command: "serve", ...buildPayload(initialResult.build) }, }); - const { databaseConfig, optionsConfig, schema, app } = initialResult.build; + const { databaseConfig, optionsConfig, schema, apps } = initialResult.build; common.options = { ...common.options, ...optionsConfig }; @@ -108,7 +108,9 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { db: database.readonlyDb, }); - const server = await createServer({ app, readonlyStore, schema, common }); + // TODO(kyle) define query + + const server = await createServer({ apps, readonlyStore, schema, common }); server.setHealthy(); cleanupReloadable = async () => { diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 3f79e6ece..40f22e8cc 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -103,7 +103,7 @@ export async function run({ }); const server = await createServer({ - app: build.app, + apps: build.apps, readonlyStore, schema, query: (...query: any) => { diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 7d91abb69..5749fe473 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -31,7 +31,8 @@ const BUILD_ID_VERSION = "1"; export type Service = { // static common: Common; - srcRegex: RegExp; + indexingRegex: RegExp; + serverRegex: RegExp; // vite viteDevServer: ViteDevServer; @@ -53,7 +54,7 @@ export type Build = { // Indexing functions indexingFunctions: IndexingFunctions; // Server - app?: Hono; + apps?: Hono[]; }; export type BuildResult = @@ -67,7 +68,7 @@ type RawBuild = { indexingFunctions: RawIndexingFunctions; contentHash: string; }; - server: { app?: Hono }; + server: { apps?: Hono[] }; }; export const create = async ({ @@ -76,12 +77,20 @@ export const create = async ({ common: Common; }): Promise => { const escapeRegex = /[.*+?^${}()|[\]\\]/g; - const escapedSrcDir = common.options.srcDir + + const escapedIndexingDir = common.options.indexingDir + // If on Windows, use a POSIX path for this regex. + .replace(/\\/g, "/") + // Escape special characters in the path. + .replace(escapeRegex, "\\$&"); + const indexingRegex = new RegExp(`^${escapedIndexingDir}/.*\\.(ts|js)$`); + + const escapedServerDir = common.options.serverDir // If on Windows, use a POSIX path for this regex. .replace(/\\/g, "/") // Escape special characters in the path. .replace(escapeRegex, "\\$&"); - const srcRegex = new RegExp(`^${escapedSrcDir}/.*\\.(ts|js)$`); + const serverRegex = new RegExp(`^${escapedServerDir}/.*\\.(ts|js)$`); const viteLogger = { warnedMessages: new Set(), @@ -133,7 +142,8 @@ export const create = async ({ return { common, - srcRegex, + indexingRegex, + serverRegex, viteDevServer, viteNodeServer, viteNodeRunner, @@ -235,10 +245,10 @@ export const start = async ( common.options.schemaFile.replace(/\\/g, "/"), ); const hasIndexingFunctionUpdate = invalidated.some((file) => - buildService.srcRegex.test(file), + buildService.indexingRegex.test(file), ); const hasServerUpdate = invalidated.includes( - common.options.serverFile.replace(/\\/g, "/"), + common.options.serverDir.replace(/\\/g, "/"), ); // This branch could trigger if you change a `note.txt` file within `src/`. @@ -382,7 +392,7 @@ const executeIndexingFunctions = async ( | { status: "error"; error: Error } > => { const pattern = path - .join(buildService.common.options.srcDir, "**/*.{js,mjs,ts,mts}") + .join(buildService.common.options.indexingDir, "**/*.{js,mjs,ts,mts}") .replace(/\\/g, "/"); const files = glob.sync(pattern); @@ -397,8 +407,6 @@ const executeIndexingFunctions = async ( for (const executeResult of executeResults) { if (executeResult.status === "error") { - console.log("Bad"); - buildService.common.logger.error({ service: "build", msg: `Error while executing '${path.relative( @@ -439,43 +447,59 @@ const executeServer = async ( ): Promise< | { status: "success"; - app?: Hono; + apps?: Hono[]; } | { status: "error"; error: Error } > => { - const doesServerExist = fs.existsSync(buildService.common.options.serverFile); + const doesServerExist = fs.existsSync(buildService.common.options.serverDir); if (doesServerExist === false) { return { status: "success" }; } - const executeResult = await executeFile(buildService, { - file: buildService.common.options.serverFile, - }); - - if (executeResult.status === "error") { - buildService.common.logger.error({ - service: "build", - msg: `Error while executing '${path.relative( - buildService.common.options.rootDir, - buildService.common.options.serverFile, - )}':`, - error: executeResult.error, - }); + const pattern = path + .join(buildService.common.options.serverDir, "**/*.{js,mjs,ts,mts}") + .replace(/\\/g, "/"); + const files = glob.sync(pattern); - return executeResult; + if (files.length === 0) { + return { status: "success" }; } - const app = executeResult.exports.hono as Hono; + const executeResults = await Promise.all( + files.map(async (file) => ({ + ...(await executeFile(buildService, { file })), + file, + })), + ); + + const apps: Hono[] = []; - buildService.common.logger.debug({ - service: "build", - msg: `Detected a custom server with routes: [${app.routes - .map((r) => r.path) - .join(", ")}]`, - }); + for (const executeResult of executeResults) { + if (executeResult.status === "error") { + buildService.common.logger.error({ + service: "build", + msg: `Error while executing '${path.relative( + buildService.common.options.rootDir, + executeResult.file, + )}':`, + error: executeResult.error, + }); + + return executeResult; + } + + apps.push(executeResult.exports?.hono as Hono); + } + + // buildService.common.logger.debug({ + // service: "build", + // msg: `Detected a custom server with routes: [${app.routes + // .map((r) => r.path) + // .join(", ")}]`, + // }); - return { status: "success", app }; + return { status: "success", apps }; }; const validateAndBuild = async ( @@ -548,7 +572,7 @@ const validateAndBuild = async ( graphQLSchema, indexingFunctions: buildConfigAndIndexingFunctionsResult.indexingFunctions, - app: rawBuild.server.app, + apps: rawBuild.server.apps, }, }; }; diff --git a/packages/core/src/common/options.ts b/packages/core/src/common/options.ts index fef3b4271..090218bc9 100644 --- a/packages/core/src/common/options.ts +++ b/packages/core/src/common/options.ts @@ -8,9 +8,9 @@ export type Options = { configFile: string; schemaFile: string; - serverFile: string; rootDir: string; - srcDir: string; + indexingDir: string; + serverDir: string; generatedDir: string; ponderDir: string; logDir: string; @@ -77,8 +77,8 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => { rootDir, configFile: path.join(rootDir, cliOptions.config), schemaFile: path.join(rootDir, "ponder.schema.ts"), - serverFile: path.join(rootDir, "ponder.server.ts"), - srcDir: path.join(rootDir, "src"), + indexingDir: path.join(rootDir, "src", "indexing"), + serverDir: path.join(rootDir, "src", "server"), generatedDir: path.join(rootDir, "generated"), ponderDir: path.join(rootDir, ".ponder"), logDir: path.join(rootDir, ".ponder", "logs"), diff --git a/packages/core/src/graphql/middleware.test.ts b/packages/core/src/graphql/index.test.ts similarity index 94% rename from packages/core/src/graphql/middleware.test.ts rename to packages/core/src/graphql/index.test.ts index 30453bbf9..a6e80ea11 100644 --- a/packages/core/src/graphql/middleware.test.ts +++ b/packages/core/src/graphql/index.test.ts @@ -10,7 +10,7 @@ import { encodeCheckpoint, zeroCheckpoint } from "@/utils/checkpoint.js"; import { Hono } from "hono"; import { createMiddleware } from "hono/factory"; import { beforeEach, expect, test } from "vitest"; -import { graphQLMiddleware } from "./middleware.js"; +import { graphql } from "./index.js"; beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); @@ -58,7 +58,7 @@ test("graphQLMiddleware serves request", async (context) => { const app = new Hono() .use(contextMiddleware(schema, readonlyStore)) - .use("/graphql", graphQLMiddleware()); + .use("/graphql", graphql()); const response = await app.request("/graphql", { method: "POST", @@ -120,7 +120,7 @@ test("graphQLMiddleware throws error when extra filter is applied", async (conte const app = new Hono() .use(contextMiddleware(schema, readonlyStore)) - .use("/graphql", graphQLMiddleware()); + .use("/graphql", graphql()); const response = await app.request("/graphql", { method: "POST", @@ -162,7 +162,7 @@ test("graphQLMiddleware throws error for token limit", async (context) => { const app = new Hono() .use(contextMiddleware(schema, readonlyStore)) - .use("/graphql", graphQLMiddleware({ maxOperationTokens: 3 })); + .use("/graphql", graphql({ maxOperationTokens: 3 })); const response = await app.request("/graphql", { method: "POST", @@ -208,7 +208,7 @@ test("graphQLMiddleware throws error for depth limit", async (context) => { const app = new Hono() .use(contextMiddleware(schema, readonlyStore)) - .use("/graphql", graphQLMiddleware({ maxOperationDepth: 5 })); + .use("/graphql", graphql({ maxOperationDepth: 5 })); const response = await app.request("/graphql", { method: "POST", @@ -254,7 +254,7 @@ test("graphQLMiddleware throws error for max aliases", async (context) => { const app = new Hono() .use(contextMiddleware(schema, readonlyStore)) - .use("/graphql", graphQLMiddleware({ maxOperationAliases: 2 })); + .use("/graphql", graphql({ maxOperationAliases: 2 })); const response = await app.request("/graphql", { method: "POST", @@ -306,7 +306,7 @@ test("graphQLMiddleware interactive", async (context) => { const app = new Hono() .use(contextMiddleware({}, readonlyStore)) - .use("/graphql", graphQLMiddleware({ maxOperationAliases: 2 })); + .use("/graphql", graphql({ maxOperationAliases: 2 })); const response = await app.request("/graphql"); diff --git a/packages/core/src/graphql/middleware.ts b/packages/core/src/graphql/index.ts similarity index 98% rename from packages/core/src/graphql/middleware.ts rename to packages/core/src/graphql/index.ts index f2873064d..972e266db 100644 --- a/packages/core/src/graphql/middleware.ts +++ b/packages/core/src/graphql/index.ts @@ -7,7 +7,7 @@ import { createMiddleware } from "hono/factory"; import { buildGraphQLSchema } from "./buildGraphqlSchema.js"; import { buildLoaderCache } from "./buildLoaderCache.js"; -export const graphQLMiddleware = ( +export const graphql = ( { maxOperationTokens = 1000, maxOperationDepth = 100, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8ce9ac7cf..114521a36 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -19,4 +19,4 @@ export type NetworkConfig = Prettify; export type BlockConfig = Prettify; export type DatabaseConfig = Prettify; -export { graphQLMiddleware } from "@/graphql/middleware.js"; +export { graphql } from "@/graphql/index.js"; diff --git a/packages/core/src/indexing/addStackTrace.ts b/packages/core/src/indexing/addStackTrace.ts index fd10a80d9..af8ccb5c5 100644 --- a/packages/core/src/indexing/addStackTrace.ts +++ b/packages/core/src/indexing/addStackTrace.ts @@ -13,12 +13,12 @@ export const addStackTrace = (error: Error, options: Options) => { // Find first frame that occurred within user code. const firstUserFrameIndex = stackTrace.findIndex((frame) => - frame.file?.includes(options.srcDir), + frame.file?.includes(options.indexingDir), ); if (firstUserFrameIndex >= 0) { userStackTrace = stackTrace.filter((frame) => - frame.file?.includes(options.srcDir), + frame.file?.includes(options.indexingDir), ); const firstUserFrame = stackTrace[firstUserFrameIndex]; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index ae135524b..1d6510dff 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -21,13 +21,13 @@ type Server = { }; export async function createServer({ - app, + apps: userApps, schema, readonlyStore, query, common, }: { - app?: Hono; + apps?: Hono[]; schema: Schema; readonlyStore: ReadonlyStore; query: (query: string) => Promise>; @@ -154,8 +154,12 @@ export async function createServer({ .route("/_ponder", ponderApp) .use(contextMiddleware); - if (app !== undefined) { - hono.route("/", app); + // TODO(kyle) check user routes + + if (userApps !== undefined) { + for (const app of userApps) { + hono.route("/", app); + } } // Create nodejs server From b88e6dde018325cd6d027eaf77559bb5064a1b25 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 18 Jun 2024 13:22:57 -0400 Subject: [PATCH 025/122] ponder.get --- packages/core/src/build/plugin.ts | 27 ++++++++----------- packages/core/src/build/service.ts | 2 +- packages/core/src/common/codegen.ts | 2 -- packages/core/src/types/virtual.ts | 40 ++++++++++++++--------------- 4 files changed, 31 insertions(+), 40 deletions(-) diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index bf6085fe2..5669ce4d3 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -4,10 +4,13 @@ import type { Plugin } from "vite"; export const ponderRegex = /^import\s+\{[^}]*\bponder\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; -export const serverRegex = - /^import\s+\{[^}]*\bhono\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; - -export const ponderShim = `export let ponder = { +export const shim = `import { Hono } from "hono"; +let __hono__ = new Hono(); +export let ponder = { + hono: __hono__, + get: __hono__.get, + post: __hono__.get, + use: __hono__.use, fns: [], on(name, fn) { this.fns.push({ name, fn }); @@ -15,10 +18,6 @@ export const ponderShim = `export let ponder = { }; `; -export const serverShim = `import { Hono } from "hono"; -export let hono = new Hono(); -`; - export function replaceStateless(code: string, regex: RegExp, shim: string) { const s = new MagicString(code); // MagicString.replace calls regex.exec(), which increments `lastIndex` @@ -33,19 +32,13 @@ export const vitePluginPonder = (): Plugin => { return { name: "ponder", transform: (code, id) => { - if (serverRegex.test(code)) { - const s = replaceStateless(code, serverRegex, serverShim); - const transformed = s.toString(); - const sourcemap = s.generateMap({ source: id }); - return { code: transformed, map: sourcemap }; - } else if (ponderRegex.test(code)) { - const s = replaceStateless(code, ponderRegex, ponderShim); + if (ponderRegex.test(code)) { + const s = replaceStateless(code, ponderRegex, shim); const transformed = s.toString(); const sourcemap = s.generateMap({ source: id }); return { code: transformed, map: sourcemap }; - } else { - return null; } + return null; }, }; }; diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 5749fe473..4835d1304 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -489,7 +489,7 @@ const executeServer = async ( return executeResult; } - apps.push(executeResult.exports?.hono as Hono); + apps.push(executeResult.exports?.ponder.hono as Hono); } // buildService.common.logger.debug({ diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index 13d0d27e8..c5149b466 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -16,8 +16,6 @@ declare module "@/generated" { export const ponder: Virtual.Registry; - export const hono: Virtual.App; - export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< config, diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index 438e585ae..0e9784eac 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -157,26 +157,22 @@ export namespace Virtual { >[property], > = ([base] extends [never] ? undefined : base) | override; - export type App = Hono<{ - Variables: { - db: { - [key in keyof InferSchemaType]: Prettify< - Pick< - DatabaseModel< - // @ts-ignore - InferSchemaType[key] - >, - "findUnique" | "findMany" - > - >; - } & { - query: ( - sqlFragment: string, - ...parameters: unknown[] - ) => Promise>; - }; - }; - }>; + type ReadonlyDb = { + [key in keyof InferSchemaType]: Prettify< + Pick< + DatabaseModel< + // @ts-ignore + InferSchemaType[key] + >, + "findUnique" | "findMany" + > + >; + } & { + query: ( + sqlFragment: string, + ...parameters: unknown[] + ) => Promise>; + }; export type Context< config extends Config, @@ -269,5 +265,9 @@ export namespace Virtual { }, ) => Promise | void, ) => void; + get: Hono<{ Variables: { db: ReadonlyDb } }>["get"]; + post: Hono<{ Variables: { db: ReadonlyDb } }>["post"]; + use: Hono<{ Variables: { db: ReadonlyDb } }>["use"]; + hono: Hono<{ Variables: { db: ReadonlyDb } }>; }; } From 350ff2801cbecc90874ad4ccb8908d34fddae07e Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 18 Jun 2024 14:55:47 -0400 Subject: [PATCH 026/122] hono peer dep --- examples/reference-erc20/ponder-env.d.ts | 2 -- examples/reference-erc20/src/server/index.ts | 6 +++--- package.json | 1 + packages/core/package.json | 2 +- pnpm-lock.yaml | 17 +++++++++++------ 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index ebd383c1d..1d5dc7ebe 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -11,8 +11,6 @@ declare module "@/generated" { export const ponder: Virtual.Registry; - export const hono: Virtual.App; - export type EventNames = Virtual.EventNames; export type Event = Virtual.Event< config, diff --git a/examples/reference-erc20/src/server/index.ts b/examples/reference-erc20/src/server/index.ts index 2b589d8ec..074fe738d 100644 --- a/examples/reference-erc20/src/server/index.ts +++ b/examples/reference-erc20/src/server/index.ts @@ -1,9 +1,9 @@ -import { hono } from "@/generated"; +import { ponder } from "@/generated"; import { graphql } from "@ponder/core"; -hono.use("/graphql", graphql()); +ponder.use("/graphql", graphql()); -hono.get("/router", async (c) => { +ponder.get("/router", async (c) => { const db = c.get("db"); // await db.query(`UPDATE "Account" SET "isOwner" = 1`); diff --git a/package.json b/package.json index ddf0782e4..769244128 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@biomejs/biome": "^1.8.1", "@changesets/changelog-github": "^0.4.8", "@changesets/cli": "^2.26.2", + "hono": "4.0.0", "lint-staged": "^15.1.0", "simple-git-hooks": "^2.9.0", "typescript": "5.0.4", diff --git a/packages/core/package.json b/packages/core/package.json index 3a13e02ce..ee3f0e7e1 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -33,6 +33,7 @@ "typecheck": "tsc --noEmit" }, "peerDependencies": { + "hono": ">=4", "typescript": ">=5.0.4", "viem": ">=1.16" }, @@ -61,7 +62,6 @@ "glob": "^10.3.10", "graphql": "^16.8.1", "graphql-yoga": "^5.3.0", - "hono": "^4.4.2", "http-terminator": "^3.2.0", "ink": "^4.4.1", "kysely": "^0.26.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb8fc1577..a7a753e83 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,6 +22,9 @@ importers: '@changesets/cli': specifier: ^2.26.2 version: 2.27.1 + hono: + specifier: 4.0.0 + version: 4.0.0 lint-staged: specifier: ^15.1.0 version: 15.2.0 @@ -440,7 +443,7 @@ importers: dependencies: forge-std: specifier: github:foundry-rs/forge-std - version: github.com/foundry-rs/forge-std/f4a0353eaabd14128fe86f4a2b9a07b827f76e46 + version: github.com/foundry-rs/forge-std/19891e6a0b5474b9ea6827ddb90bb9388f7acfc0 examples/with-foundry/ponder: dependencies: @@ -617,9 +620,6 @@ importers: graphql-yoga: specifier: ^5.3.0 version: 5.3.1(graphql@16.8.2) - hono: - specifier: ^4.4.2 - version: 4.4.6 http-terminator: specifier: ^3.2.0 version: 3.2.0 @@ -8529,6 +8529,11 @@ packages: minimalistic-crypto-utils: 1.0.1 dev: true + /hono@4.0.0: + resolution: {integrity: sha512-8dKhuBBpRZEodUttQhrSFJ6PQqHRjXHyeeegfxOf132pvgbf0tOb9qqb7q7eYwAWpOcYrsUOsWdJ0sQIIovhZg==} + engines: {node: '>=16.0.0'} + dev: true + /hono@4.4.6: resolution: {integrity: sha512-XGRnoH8WONv60+PPvP9Sn067A9r/8JdHDJ5bgon0DVEHeR1cJPkWjv2aT+DBfMH9/mEkYa1+VEVFp1DT1lIwjw==} engines: {node: '>=16.0.0'} @@ -14965,8 +14970,8 @@ packages: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} dev: false - github.com/foundry-rs/forge-std/f4a0353eaabd14128fe86f4a2b9a07b827f76e46: - resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/f4a0353eaabd14128fe86f4a2b9a07b827f76e46} + github.com/foundry-rs/forge-std/19891e6a0b5474b9ea6827ddb90bb9388f7acfc0: + resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/19891e6a0b5474b9ea6827ddb90bb9388f7acfc0} name: forge-std version: 1.8.2 dev: false From d9bb14141b7185407c661755ab9c3e3208dd6313 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 18 Jun 2024 15:03:14 -0400 Subject: [PATCH 027/122] fix tests --- .../core/src/_test/e2e/erc20/src/_server.ts | 4 -- .../e2e/erc20/src/{ => indexing}/index.ts | 4 +- .../src/_test/e2e/erc20/src/server/index.ts | 11 ++++++ .../core/src/_test/e2e/factory/src/_server.ts | 4 -- .../e2e/factory/src/{ => indexing}/index.ts | 4 +- .../src/_test/e2e/factory/src/server/index.ts | 11 ++++++ packages/core/src/bin/commands/serve.ts | 15 ++++++-- packages/core/src/build/plugin.test.ts | 38 +++++++++---------- packages/core/src/server/service.test.ts | 20 ++++++++++ 9 files changed, 77 insertions(+), 34 deletions(-) delete mode 100644 packages/core/src/_test/e2e/erc20/src/_server.ts rename packages/core/src/_test/e2e/erc20/src/{ => indexing}/index.ts (88%) create mode 100644 packages/core/src/_test/e2e/erc20/src/server/index.ts delete mode 100644 packages/core/src/_test/e2e/factory/src/_server.ts rename packages/core/src/_test/e2e/factory/src/{ => indexing}/index.ts (79%) create mode 100644 packages/core/src/_test/e2e/factory/src/server/index.ts diff --git a/packages/core/src/_test/e2e/erc20/src/_server.ts b/packages/core/src/_test/e2e/erc20/src/_server.ts deleted file mode 100644 index 613324c36..000000000 --- a/packages/core/src/_test/e2e/erc20/src/_server.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { hono } from "@/generated"; -import { graphQLMiddleware } from "@/index.js"; - -hono.use("/graphql", graphQLMiddleware()); diff --git a/packages/core/src/_test/e2e/erc20/src/index.ts b/packages/core/src/_test/e2e/erc20/src/indexing/index.ts similarity index 88% rename from packages/core/src/_test/e2e/erc20/src/index.ts rename to packages/core/src/_test/e2e/erc20/src/indexing/index.ts index 029728b8a..27cd8f8b1 100644 --- a/packages/core/src/_test/e2e/erc20/src/index.ts +++ b/packages/core/src/_test/e2e/erc20/src/indexing/index.ts @@ -3,8 +3,8 @@ import { ponder } from "@/generated"; // biome-ignore lint/suspicious/noRedeclare: :) declare const ponder: import("@/index.js").Virtual.Registry< - typeof import("../ponder.config.js").default, - typeof import("../ponder.schema.js").default + typeof import("../../ponder.config.js").default, + typeof import("../../ponder.schema.js").default >; ponder.on( diff --git a/packages/core/src/_test/e2e/erc20/src/server/index.ts b/packages/core/src/_test/e2e/erc20/src/server/index.ts new file mode 100644 index 000000000..5cf910996 --- /dev/null +++ b/packages/core/src/_test/e2e/erc20/src/server/index.ts @@ -0,0 +1,11 @@ +// @ts-ignore +import { ponder } from "@/generated"; +import { graphql } from "@/index.js"; + +// biome-ignore lint/suspicious/noRedeclare: :) +declare const ponder: import("@/index.js").Virtual.Registry< + typeof import("../../ponder.config.js").default, + typeof import("../../ponder.schema.js").default +>; + +ponder.use("/graphql", graphql()); diff --git a/packages/core/src/_test/e2e/factory/src/_server.ts b/packages/core/src/_test/e2e/factory/src/_server.ts deleted file mode 100644 index 613324c36..000000000 --- a/packages/core/src/_test/e2e/factory/src/_server.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { hono } from "@/generated"; -import { graphQLMiddleware } from "@/index.js"; - -hono.use("/graphql", graphQLMiddleware()); diff --git a/packages/core/src/_test/e2e/factory/src/index.ts b/packages/core/src/_test/e2e/factory/src/indexing/index.ts similarity index 79% rename from packages/core/src/_test/e2e/factory/src/index.ts rename to packages/core/src/_test/e2e/factory/src/indexing/index.ts index 15e36a2e6..9123e6144 100644 --- a/packages/core/src/_test/e2e/factory/src/index.ts +++ b/packages/core/src/_test/e2e/factory/src/indexing/index.ts @@ -3,8 +3,8 @@ import { ponder } from "@/generated"; // biome-ignore lint/suspicious/noRedeclare: :) declare const ponder: import("@/index.js").Virtual.Registry< - typeof import("../ponder.config.js").default, - typeof import("../ponder.schema.js").default + typeof import("../../ponder.config.js").default, + typeof import("../../ponder.schema.js").default >; ponder.on("Pair:Swap", async ({ event, context }) => { diff --git a/packages/core/src/_test/e2e/factory/src/server/index.ts b/packages/core/src/_test/e2e/factory/src/server/index.ts new file mode 100644 index 000000000..5cf910996 --- /dev/null +++ b/packages/core/src/_test/e2e/factory/src/server/index.ts @@ -0,0 +1,11 @@ +// @ts-ignore +import { ponder } from "@/generated"; +import { graphql } from "@/index.js"; + +// biome-ignore lint/suspicious/noRedeclare: :) +declare const ponder: import("@/index.js").Virtual.Registry< + typeof import("../../ponder.config.js").default, + typeof import("../../ponder.schema.js").default +>; + +ponder.use("/graphql", graphql()); diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index a74acd4a4..6ada838b1 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -8,6 +8,7 @@ import { PostgresDatabaseService } from "@/database/postgres/service.js"; import type { NamespaceInfo } from "@/database/service.js"; import { getReadonlyStore } from "@/indexing-store/readonly.js"; import { createServer } from "@/server/service.js"; +import { sql } from "kysely"; import type { CliOptions } from "../ponder.js"; import { setupShutdown } from "../utils/shutdown.js"; @@ -108,9 +109,17 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { db: database.readonlyDb, }); - // TODO(kyle) define query - - const server = await createServer({ apps, readonlyStore, schema, common }); + const server = await createServer({ + apps, + readonlyStore, + schema, + query: (...query: any) => { + return sql + .raw(query) + .execute(database.readonlyDb.withSchema(userNamespace)); + }, + common, + }); server.setHealthy(); cleanupReloadable = async () => { diff --git a/packages/core/src/build/plugin.test.ts b/packages/core/src/build/plugin.test.ts index c44570c06..82f62ca8b 100644 --- a/packages/core/src/build/plugin.test.ts +++ b/packages/core/src/build/plugin.test.ts @@ -1,12 +1,12 @@ import { expect, test } from "vitest"; -import { ponderRegex, ponderShim, replaceStateless } from "./plugin.js"; +import { ponderRegex, replaceStateless, shim } from "./plugin.js"; test("regex matches basic", () => { const code = `import { ponder } from "@/generated";\n`; expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, ponderShim); - expect(s.toString().includes(ponderShim)).toBe(true); + const s = replaceStateless(code, ponderRegex, shim); + expect(s.toString().includes(shim)).toBe(true); }); test("regex matches multiline", () => { @@ -15,24 +15,24 @@ test("regex matches multiline", () => { 'ponder.on("PrimitiveManager:Swap", async ({ event, context }) => {\n'; expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, ponderShim); - expect(s.toString().includes(ponderShim)).toBe(true); + const s = replaceStateless(code, ponderRegex, shim); + expect(s.toString().includes(shim)).toBe(true); }); test("regex matches import including types before", () => { const code = 'import { type Context, ponder } from "@/generated";\n'; expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, ponderShim); - expect(s.toString().includes(ponderShim)).toBe(true); + const s = replaceStateless(code, ponderRegex, shim); + expect(s.toString().includes(shim)).toBe(true); }); test("regex matches import includinga types after", () => { const code = 'import { ponder, type Context } from "@/generated";\n'; expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, ponderShim); - expect(s.toString().includes(ponderShim)).toBe(true); + const s = replaceStateless(code, ponderRegex, shim); + expect(s.toString().includes(shim)).toBe(true); }); test("regex matches import including newlines", () => { @@ -40,32 +40,32 @@ test("regex matches import including newlines", () => { "import {\n" + "ponder,\n" + "type Context,\n" + '} from "@/generated";\n'; expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, ponderShim); - expect(s.toString().includes(ponderShim)).toBe(true); + const s = replaceStateless(code, ponderRegex, shim); + expect(s.toString().includes(shim)).toBe(true); }); test("regex matches no trailing semicolon", () => { const code = `import { ponder } from "@/generated"`; expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, ponderShim); - expect(s.toString().includes(ponderShim)).toBe(true); + const s = replaceStateless(code, ponderRegex, shim); + expect(s.toString().includes(shim)).toBe(true); }); test("regex matches no trailing single quote import", () => { const code = `import { ponder } from '@/generated'`; expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, ponderShim); - expect(s.toString().includes(ponderShim)).toBe(true); + const s = replaceStateless(code, ponderRegex, shim); + expect(s.toString().includes(shim)).toBe(true); }); test("regex matches no trailing newline", () => { const code = `import { ponder } from "@/generated";ponder.on("PrimitiveManager:Swap", async ({ event, context }) => {`; expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, ponderShim); - expect(s.toString().includes(ponderShim)).toBe(true); + const s = replaceStateless(code, ponderRegex, shim); + expect(s.toString().includes(shim)).toBe(true); }); test("regex matches preceding import", () => { @@ -74,6 +74,6 @@ test("regex matches preceding import", () => { `import {ponder} from "@/generated";\n`; expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, ponderShim); - expect(s.toString().includes(ponderShim)).toBe(true); + const s = replaceStateless(code, ponderRegex, shim); + expect(s.toString().includes(shim)).toBe(true); }); diff --git a/packages/core/src/server/service.test.ts b/packages/core/src/server/service.test.ts index 363ee1945..4cd50d8d6 100644 --- a/packages/core/src/server/service.test.ts +++ b/packages/core/src/server/service.test.ts @@ -11,12 +11,16 @@ test("port", async (context) => { const server1 = await createServer({ schema: {} as Schema, common: context.common, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); const server2 = await createServer({ schema: {} as Schema, common: context.common, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); @@ -33,6 +37,8 @@ test("not healthy", async (context) => { ...context.common, options: { ...context.common.options, maxHealthcheckDuration: 5 }, }, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); @@ -50,6 +56,8 @@ test("healthy", async (context) => { ...context.common, options: { ...context.common.options, maxHealthcheckDuration: 0 }, }, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); @@ -67,6 +75,8 @@ test("healthy PUT", async (context) => { ...context.common, options: { ...context.common.options, maxHealthcheckDuration: 0 }, }, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); @@ -83,6 +93,8 @@ test("metrics", async (context) => { const server = await createServer({ schema: {} as Schema, common: context.common, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); @@ -97,6 +109,8 @@ test("metrics error", async (context) => { const server = await createServer({ schema: {} as Schema, common: context.common, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); @@ -114,6 +128,8 @@ test("metrics PUT", async (context) => { const server = await createServer({ schema: {} as Schema, common: context.common, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); @@ -130,6 +146,8 @@ test("missing route", async (context) => { const server = await createServer({ schema: {} as Schema, common: context.common, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); @@ -146,6 +164,8 @@ test.skip("kill", async (context) => { const server = await createServer({ schema: {} as Schema, common: context.common, + // @ts-ignore + query: undefined, readonlyStore: {} as ReadonlyStore, }); From 210adb79ce3b2a618ba02098086446d1fd611118 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 18 Jun 2024 15:27:39 -0400 Subject: [PATCH 028/122] more logs --- packages/core/src/build/service.ts | 13 ++++------ packages/core/src/server/service.ts | 39 ++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 4835d1304..3145a52c5 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -489,15 +489,12 @@ const executeServer = async ( return executeResult; } - apps.push(executeResult.exports?.ponder.hono as Hono); - } + if (executeResult.exports?.ponder?.hono === undefined) { + continue; + } - // buildService.common.logger.debug({ - // service: "build", - // msg: `Detected a custom server with routes: [${app.routes - // .map((r) => r.path) - // .join(", ")}]`, - // }); + apps.push(executeResult.exports.ponder.hono as Hono); + } return { status: "success", apps }; }; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 1d6510dff..aedccc14e 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -154,7 +154,44 @@ export async function createServer({ .route("/_ponder", ponderApp) .use(contextMiddleware); - // TODO(kyle) check user routes + if (userApps !== undefined) { + const routes = new Set(); + + for (const app of userApps) { + for (const route of app.routes) { + // Validate user routes don't conflict with ponder routes + if ( + route.path === "/_ponder/metrics" || + route.path === "/_ponder/health" + ) { + common.logger.warn({ + service: "server", + msg: `Ingoring path '${route.path}' with method '${route.method}' because '/_ponder' is reserved`, + }); + } + + // Validate user routes don't conflict have duplicates + const _route = `${route.method}_${route.path}`; + if (routes.has(_route)) { + common.logger.warn({ + service: "server", + msg: `Path '${route.path}' with method '${route.method}' already has been defined`, + }); + } else { + routes.add(_route); + } + } + + hono.route("/", app); + } + + common.logger.debug({ + service: "server", + msg: `Detected a custom server with routes: [${userApps + .flatMap((app) => app.routes.map((r) => r.path)) + .join(", ")}]`, + }); + } if (userApps !== undefined) { for (const app of userApps) { From 1bb9f575a9f0bab681a90a043f536bb5a1d5a861 Mon Sep 17 00:00:00 2001 From: kyscott18 <43524469+kyscott18@users.noreply.github.com> Date: Tue, 18 Jun 2024 17:57:13 -0400 Subject: [PATCH 029/122] dynamic batch size based on number of columns (#944) * dynamic batch size based on number of params * use option for max_batch_size * chore: changeset --- .changeset/four-plums-invite.md | 5 ++ packages/core/src/_test/setup.ts | 4 ++ packages/core/src/bin/commands/serve.ts | 1 + packages/core/src/bin/utils/run.ts | 2 + packages/core/src/common/errors.ts | 9 ++++ packages/core/src/common/options.ts | 5 ++ .../src/database/postgres/service.test.ts | 2 + .../core/src/database/sqlite/service.test.ts | 2 + .../core/src/indexing-store/historical.ts | 46 +++++++++++-------- packages/core/src/indexing-store/readonly.ts | 8 ++-- packages/core/src/indexing-store/realtime.ts | 14 ++++-- 11 files changed, 72 insertions(+), 26 deletions(-) create mode 100644 .changeset/four-plums-invite.md diff --git a/.changeset/four-plums-invite.md b/.changeset/four-plums-invite.md new file mode 100644 index 000000000..804e5ab54 --- /dev/null +++ b/.changeset/four-plums-invite.md @@ -0,0 +1,5 @@ +--- +"@ponder/core": patch +--- + +Fixed "too many SQL variables" bug for tables with >30 columns. diff --git a/packages/core/src/_test/setup.ts b/packages/core/src/_test/setup.ts index 51693f2e8..3c29fd938 100644 --- a/packages/core/src/_test/setup.ts +++ b/packages/core/src/_test/setup.ts @@ -165,6 +165,7 @@ export async function setupDatabaseServices( schema: config.schema, namespaceInfo: result.namespaceInfo, db: database.indexingDb, + common: context.common, }); const indexingStore = @@ -185,6 +186,7 @@ export async function setupDatabaseServices( schema: config.schema, namespaceInfo: result.namespaceInfo, db: database.indexingDb, + common: context.common, }), }; @@ -218,6 +220,7 @@ export async function setupDatabaseServices( schema: config.schema, namespaceInfo: result.namespaceInfo, db: database.indexingDb, + common: context.common, }); const indexingStore = @@ -238,6 +241,7 @@ export async function setupDatabaseServices( schema: config.schema, namespaceInfo: result.namespaceInfo, db: database.indexingDb, + common: context.common, }), }; diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index 6ada838b1..54d29c6a0 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -107,6 +107,7 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { userNamespace: databaseConfig.publishSchema, } as unknown as NamespaceInfo, db: database.readonlyDb, + common, }); const server = await createServer({ diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 40f22e8cc..b562318da 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -100,6 +100,7 @@ export async function run({ schema, namespaceInfo, db: database.readonlyDb, + common, }); const server = await createServer({ @@ -291,6 +292,7 @@ export async function run({ schema, namespaceInfo, db: database.indexingDb, + common, }), }; diff --git a/packages/core/src/common/errors.ts b/packages/core/src/common/errors.ts index a95cf1d5c..c7689ba47 100644 --- a/packages/core/src/common/errors.ts +++ b/packages/core/src/common/errors.ts @@ -99,3 +99,12 @@ export class BigIntSerializationError extends NonRetryableError { Object.setPrototypeOf(this, BigIntSerializationError.prototype); } } + +export class FlushError extends NonRetryableError { + override name = "FlushError"; + + constructor(message?: string | undefined) { + super(message); + Object.setPrototypeOf(this, FlushError.prototype); + } +} diff --git a/packages/core/src/common/options.ts b/packages/core/src/common/options.ts index 090218bc9..ed95747e8 100644 --- a/packages/core/src/common/options.ts +++ b/packages/core/src/common/options.ts @@ -28,6 +28,8 @@ export type Options = { databaseHeartbeatInterval: number; databaseHeartbeatTimeout: number; + databaseMaxQueryParameters: number; + databaseMaxRowLimit: number; indexingCacheMaxBytes: number; indexingCacheFlushRatio: number; @@ -96,6 +98,9 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => { databaseHeartbeatInterval: 10 * 1000, databaseHeartbeatTimeout: 25 * 1000, + // Half of the max query parameters for SQLite + databaseMaxQueryParameters: 16_000, + databaseMaxRowLimit: 1_000, // os.freemem() / 4, bucketed closest to 64, 128, 256, 512, 1024, 2048 mB indexingCacheMaxBytes: diff --git a/packages/core/src/database/postgres/service.test.ts b/packages/core/src/database/postgres/service.test.ts index 35a293be7..9368298a2 100644 --- a/packages/core/src/database/postgres/service.test.ts +++ b/packages/core/src/database/postgres/service.test.ts @@ -248,6 +248,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { schema, db: database.indexingDb, namespaceInfo, + common: context.common, }); // Simulate progress being made by updating the checkpoints. @@ -311,6 +312,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { schema, db: databaseTwo.indexingDb, namespaceInfo: namespaceInfoTwo, + common: context.common, }); expect(checkpoint).toMatchObject(newCheckpoint); diff --git a/packages/core/src/database/sqlite/service.test.ts b/packages/core/src/database/sqlite/service.test.ts index 58981ad85..9251bec30 100644 --- a/packages/core/src/database/sqlite/service.test.ts +++ b/packages/core/src/database/sqlite/service.test.ts @@ -188,6 +188,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { schema, db: database.indexingDb, namespaceInfo, + common: context.common, }); // Simulate progress being made by updating the checkpoints. @@ -250,6 +251,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { schema, db: databaseTwo.indexingDb, namespaceInfo: namespaceInfoTwo, + common: context.common, }); expect(checkpoint).toMatchObject(newCheckpoint); diff --git a/packages/core/src/indexing-store/historical.ts b/packages/core/src/indexing-store/historical.ts index 928c714f3..179fc6069 100644 --- a/packages/core/src/indexing-store/historical.ts +++ b/packages/core/src/indexing-store/historical.ts @@ -1,5 +1,9 @@ import type { Common } from "@/common/common.js"; -import { RecordNotFoundError, UniqueConstraintError } from "@/common/errors.js"; +import { + FlushError, + RecordNotFoundError, + UniqueConstraintError, +} from "@/common/errors.js"; import type { HeadlessKysely } from "@/database/kysely.js"; import type { NamespaceInfo } from "@/database/service.js"; import type { Schema, Table } from "@/schema/common.js"; @@ -35,8 +39,6 @@ import { import { parseStoreError } from "./utils/errors.js"; import { buildWhereConditions } from "./utils/filter.js"; -const MAX_BATCH_SIZE = 1_000; - /** Cache entries that need to be created in the database. */ type InsertEntry = { type: "insert"; @@ -179,6 +181,10 @@ export const getHistoricalStore = ({ Object.entries(storeCache).map(async ([tableName, tableStoreCache]) => { const table = (schema[tableName] as { table: Table }).table; const cacheEntries = Object.values(tableStoreCache); + const batchSize = Math.round( + common.options.databaseMaxQueryParameters / + Object.keys(table).length, + ); let insertRecords: UserRecord[]; @@ -204,11 +210,11 @@ export const getHistoricalStore = ({ for ( let i = 0, len = insertRecords.length; i < len; - i += MAX_BATCH_SIZE + i += batchSize ) { await db.wrap({ method: `${tableName}.flush` }, async () => { const _insertRecords = insertRecords - .slice(i, i + MAX_BATCH_SIZE) + .slice(i, i + batchSize) // skip validation because its already occurred in the store method .map((record) => encodeRecord({ @@ -225,11 +231,13 @@ export const getHistoricalStore = ({ .insertInto(tableName) .values(_insertRecords) .execute() - .catch((err) => { - throw parseStoreError( - err, - _insertRecords.length > 0 ? _insertRecords[0]! : {}, - ); + .catch((_error) => { + const error = _error as Error; + common.logger.error({ + service: "indexing", + msg: "Internal error occurred while flushing cache. Please report this error here: https://github.com/ponder-sh/ponder/issues", + }); + throw new FlushError(error.message); }); }); } @@ -262,11 +270,11 @@ export const getHistoricalStore = ({ for ( let i = 0, len = updateRecords.length; i < len; - i += MAX_BATCH_SIZE + i += batchSize ) { await db.wrap({ method: `${tableName}.flush` }, async () => { const _updateRecords = updateRecords - .slice(i, i + MAX_BATCH_SIZE) + .slice(i, i + batchSize) // skip validation because its already occurred in the store method .map((record) => encodeRecord({ @@ -303,11 +311,13 @@ export const getHistoricalStore = ({ ), ) .execute() - .catch((err) => { - throw parseStoreError( - err, - _updateRecords.length > 0 ? _updateRecords[0]! : {}, - ); + .catch((_error) => { + const error = _error as Error; + common.logger.error({ + service: "indexing", + msg: "Internal error occurred while flushing cache. Please report this error here: https://github.com/ponder-sh/ponder/issues", + }); + throw new FlushError(error.message); }); }); } @@ -613,7 +623,7 @@ export const getHistoricalStore = ({ { method: `${tableName}.updateMany` }, async () => { const latestRecords: DatabaseRecord[] = await query - .limit(MAX_BATCH_SIZE) + .limit(common.options.databaseMaxRowLimit) .$if(cursor !== null, (qb) => qb.where("id", ">", cursor)) .execute(); diff --git a/packages/core/src/indexing-store/readonly.ts b/packages/core/src/indexing-store/readonly.ts index 2b1c0bc5e..33c558651 100644 --- a/packages/core/src/indexing-store/readonly.ts +++ b/packages/core/src/indexing-store/readonly.ts @@ -1,3 +1,4 @@ +import type { Common } from "@/common/common.js"; import { StoreError } from "@/common/errors.js"; import type { HeadlessKysely } from "@/database/kysely.js"; import type { NamespaceInfo } from "@/database/service.js"; @@ -18,18 +19,19 @@ import { } from "./utils/sort.js"; const DEFAULT_LIMIT = 50 as const; -const MAX_LIMIT = 1_000 as const; export const getReadonlyStore = ({ encoding, schema, namespaceInfo, db, + common, }: { encoding: "sqlite" | "postgres"; schema: Schema; namespaceInfo: NamespaceInfo; db: HeadlessKysely; + common: Common; }): ReadonlyStore => ({ findUnique: async ({ tableName, @@ -101,9 +103,9 @@ export const getReadonlyStore = ({ } const orderDirection = orderByConditions[0]![1]; - if (limit > MAX_LIMIT) { + if (limit > common.options.databaseMaxRowLimit) { throw new StoreError( - `Invalid limit. Got ${limit}, expected <=${MAX_LIMIT}.`, + `Invalid limit. Got ${limit}, expected <=${common.options.databaseMaxRowLimit}.`, ); } diff --git a/packages/core/src/indexing-store/realtime.ts b/packages/core/src/indexing-store/realtime.ts index 7d44fc7f1..d78c7aa1f 100644 --- a/packages/core/src/indexing-store/realtime.ts +++ b/packages/core/src/indexing-store/realtime.ts @@ -1,3 +1,4 @@ +import type { Common } from "@/common/common.js"; import type { HeadlessKysely } from "@/database/kysely.js"; import type { NamespaceInfo } from "@/database/service.js"; import type { Schema, Table } from "@/schema/common.js"; @@ -12,18 +13,18 @@ import { decodeRecord, encodeRecord, encodeValue } from "./utils/encoding.js"; import { parseStoreError } from "./utils/errors.js"; import { buildWhereConditions } from "./utils/filter.js"; -const MAX_BATCH_SIZE = 1_000 as const; - export const getRealtimeStore = ({ encoding, schema, namespaceInfo, db, + common, }: { encoding: "sqlite" | "postgres"; schema: Schema; namespaceInfo: NamespaceInfo; db: HeadlessKysely; + common: Common; }): WriteStore<"realtime"> => ({ create: ({ tableName, @@ -86,8 +87,11 @@ export const getRealtimeStore = ({ return db.wrap({ method: `${tableName}.createMany` }, async () => { const records: DatabaseRecord[] = []; await db.transaction().execute(async (tx) => { - for (let i = 0, len = data.length; i < len; i += MAX_BATCH_SIZE) { - const createRecords = data.slice(i, i + MAX_BATCH_SIZE).map((d) => + const batchSize = Math.round( + common.options.databaseMaxQueryParameters / Object.keys(table).length, + ); + for (let i = 0, len = data.length; i < len; i += batchSize) { + const createRecords = data.slice(i, i + batchSize).map((d) => encodeRecord({ record: d, table, @@ -238,7 +242,7 @@ export const getRealtimeStore = ({ }), ) .orderBy("id", "asc") - .limit(MAX_BATCH_SIZE) + .limit(common.options.databaseMaxRowLimit) .$if(cursor !== null, (qb) => qb.where("id", ">", cursor)) .execute(); From ee9d6aaa025692cf0dae03e61133875e0aca99b2 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 18 Jun 2024 18:37:31 -0400 Subject: [PATCH 030/122] format on lint staged --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 769244128..db36a190a 100644 --- a/package.json +++ b/package.json @@ -24,8 +24,8 @@ "viem": "1.16.0" }, "lint-staged": { - "*.ts": ["biome format --no-errors-on-unmatched", "biome check"], - "!(*.ts)": ["biome format --no-errors-on-unmatched"] + "*.ts": ["biome format --no-errors-on-unmatched --write", "biome check"], + "!(*.ts)": ["biome format --no-errors-on-unmatched --write"] }, "simple-git-hooks": { "pre-commit": "npx lint-staged" From 96f4e041fdbebdcbb6b554b4684d907d0bdf58e6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:24:39 -0400 Subject: [PATCH 031/122] chore: version packages (#945) Co-authored-by: github-actions[bot] --- .changeset/four-plums-invite.md | 5 ----- packages/core/CHANGELOG.md | 6 ++++++ packages/core/package.json | 2 +- packages/create-ponder/CHANGELOG.md | 2 ++ packages/create-ponder/package.json | 2 +- packages/eslint-config-ponder/CHANGELOG.md | 2 ++ packages/eslint-config-ponder/package.json | 2 +- 7 files changed, 13 insertions(+), 8 deletions(-) delete mode 100644 .changeset/four-plums-invite.md diff --git a/.changeset/four-plums-invite.md b/.changeset/four-plums-invite.md deleted file mode 100644 index 804e5ab54..000000000 --- a/.changeset/four-plums-invite.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@ponder/core": patch ---- - -Fixed "too many SQL variables" bug for tables with >30 columns. diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 527dc9d11..b8036ce3c 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,11 @@ # @ponder/core +## 0.4.39 + +### Patch Changes + +- [#944](https://github.com/ponder-sh/ponder/pull/944) [`7ca63a8f42c7a6650508a5c7904785c1a4780c47`](https://github.com/ponder-sh/ponder/commit/7ca63a8f42c7a6650508a5c7904785c1a4780c47) Thanks [@kyscott18](https://github.com/kyscott18)! - Fixed "too many SQL variables" bug for tables with >30 columns. + ## 0.4.38 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index ee3f0e7e1..dbdfec08d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@ponder/core", - "version": "0.4.38", + "version": "0.4.39", "description": "An open-source framework for crypto application backends", "license": "MIT", "type": "module", diff --git a/packages/create-ponder/CHANGELOG.md b/packages/create-ponder/CHANGELOG.md index 754bf1894..bae8fb176 100644 --- a/packages/create-ponder/CHANGELOG.md +++ b/packages/create-ponder/CHANGELOG.md @@ -1,5 +1,7 @@ # create-ponder +## 0.4.39 + ## 0.4.38 ## 0.4.37 diff --git a/packages/create-ponder/package.json b/packages/create-ponder/package.json index ce701d04e..2e73370f8 100644 --- a/packages/create-ponder/package.json +++ b/packages/create-ponder/package.json @@ -1,6 +1,6 @@ { "name": "create-ponder", - "version": "0.4.38", + "version": "0.4.39", "type": "module", "description": "A CLI tool to create Ponder apps", "license": "MIT", diff --git a/packages/eslint-config-ponder/CHANGELOG.md b/packages/eslint-config-ponder/CHANGELOG.md index c64a858e0..1e9e79e20 100644 --- a/packages/eslint-config-ponder/CHANGELOG.md +++ b/packages/eslint-config-ponder/CHANGELOG.md @@ -1,5 +1,7 @@ # eslint-config-ponder +## 0.4.39 + ## 0.4.38 ## 0.4.37 diff --git a/packages/eslint-config-ponder/package.json b/packages/eslint-config-ponder/package.json index e95e57470..6da0951f9 100644 --- a/packages/eslint-config-ponder/package.json +++ b/packages/eslint-config-ponder/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-ponder", - "version": "0.4.38", + "version": "0.4.39", "description": "ESLint config for Ponder apps", "license": "MIT", "main": "./index.js", From e33c7e917d93de732a9fa0944ea8bb5852395ef4 Mon Sep 17 00:00:00 2001 From: kyscott18 <43524469+kyscott18@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:11:16 -0400 Subject: [PATCH 032/122] Fix bug with material columns (#946) * filter out non-material columns * chore: changeset --- .changeset/lovely-sheep-joke.md | 5 +++++ .../core/src/indexing-store/historical.ts | 22 +++++-------------- packages/core/src/indexing-store/readonly.ts | 6 ++--- .../core/src/indexing-store/utils/encoding.ts | 14 +++++------- packages/core/src/schema/common.ts | 2 +- packages/core/src/schema/utils.ts | 8 +++++++ 6 files changed, 28 insertions(+), 29 deletions(-) create mode 100644 .changeset/lovely-sheep-joke.md diff --git a/.changeset/lovely-sheep-joke.md b/.changeset/lovely-sheep-joke.md new file mode 100644 index 000000000..998bc8a5d --- /dev/null +++ b/.changeset/lovely-sheep-joke.md @@ -0,0 +1,5 @@ +--- +"@ponder/core": patch +--- + +Fixed an indexing bug causing the "syntax error at end of input" error. diff --git a/packages/core/src/indexing-store/historical.ts b/packages/core/src/indexing-store/historical.ts index 179fc6069..f669eac74 100644 --- a/packages/core/src/indexing-store/historical.ts +++ b/packages/core/src/indexing-store/historical.ts @@ -9,8 +9,7 @@ import type { NamespaceInfo } from "@/database/service.js"; import type { Schema, Table } from "@/schema/common.js"; import { getTables, - isEnumColumn, - isJSONColumn, + isMaterialColumn, isReferenceColumn, isScalarColumn, } from "@/schema/utils.js"; @@ -146,13 +145,7 @@ export const getHistoricalStore = ({ tables[tableName]!.table, )) { // optional columns are null - if ( - (isScalarColumn(column) || - isReferenceColumn(column) || - isEnumColumn(column) || - isJSONColumn(column)) && - record[columnName] === undefined - ) { + if (isMaterialColumn(column) && record[columnName] === undefined) { record[columnName] = null; } // hex is lowercase byte encoded @@ -244,7 +237,9 @@ export const getHistoricalStore = ({ } // Exit early if the table only has an "id" column. - if (Object.keys(table).length === 1) return; + if (Object.values(table).filter(isMaterialColumn).length === 1) { + return; + } let updateRecords: UserRecord[]; @@ -295,12 +290,7 @@ export const getHistoricalStore = ({ Object.entries(table).reduce( (acc, [colName, column]) => { if (colName !== "id") { - if ( - isScalarColumn(column) || - isReferenceColumn(column) || - isEnumColumn(column) || - isJSONColumn(column) - ) { + if (isMaterialColumn(column)) { acc[colName] = eb.ref(`excluded.${colName}`); } } diff --git a/packages/core/src/indexing-store/readonly.ts b/packages/core/src/indexing-store/readonly.ts index 33c558651..790c8d1bf 100644 --- a/packages/core/src/indexing-store/readonly.ts +++ b/packages/core/src/indexing-store/readonly.ts @@ -2,7 +2,7 @@ import type { Common } from "@/common/common.js"; import { StoreError } from "@/common/errors.js"; import type { HeadlessKysely } from "@/database/kysely.js"; import type { NamespaceInfo } from "@/database/service.js"; -import type { NonVirtualColumn, Schema, Table } from "@/schema/common.js"; +import type { MaterialColumn, Schema, Table } from "@/schema/common.js"; import type { UserId } from "@/types/schema.js"; import { sql } from "kysely"; import type { OrderByInput, ReadonlyStore, WhereInput } from "./store.js"; @@ -154,7 +154,7 @@ export const getReadonlyStore = ({ columnName, encodeValue({ value, - column: table[columnName] as NonVirtualColumn, + column: table[columnName] as MaterialColumn, encoding, }), ]) satisfies [string, any][]; @@ -220,7 +220,7 @@ export const getReadonlyStore = ({ columnName, encodeValue({ value, - column: table[columnName] as NonVirtualColumn, + column: table[columnName] as MaterialColumn, encoding, }), ]) satisfies [string, any][]; diff --git a/packages/core/src/indexing-store/utils/encoding.ts b/packages/core/src/indexing-store/utils/encoding.ts index ed67ce177..5cb3f8eee 100644 --- a/packages/core/src/indexing-store/utils/encoding.ts +++ b/packages/core/src/indexing-store/utils/encoding.ts @@ -8,7 +8,7 @@ import type { Column, EnumColumn, JSONColumn, - NonVirtualColumn, + MaterialColumn, ReferenceColumn, Scalar, ScalarColumn, @@ -21,6 +21,7 @@ import { isJSONColumn, isListColumn, isManyColumn, + isMaterialColumn, isOptionalColumn, isReferenceColumn, isScalarColumn, @@ -66,7 +67,7 @@ export function encodeRecord({ // user data is considered to be valid at this point for (const [columnName, value] of Object.entries(record)) { - const column = table[columnName] as NonVirtualColumn; + const column = table[columnName] as MaterialColumn; instance[columnName] = encodeValue({ value, @@ -88,7 +89,7 @@ export function encodeValue( encoding, }: { value: UserValue; - column: NonVirtualColumn; + column: MaterialColumn; encoding: "sqlite" | "postgres"; }, // @ts-ignore @@ -351,12 +352,7 @@ export function decodeRecord({ const instance = {} as UserRecord; for (const [columnName, column] of Object.entries(table)) { - if ( - isScalarColumn(column) || - isReferenceColumn(column) || - isEnumColumn(column) || - isJSONColumn(column) - ) { + if (isMaterialColumn(column)) { instance[columnName] = decodeValue({ value: record[columnName]!, column, diff --git a/packages/core/src/schema/common.ts b/packages/core/src/schema/common.ts index 1baf331a5..17422773a 100644 --- a/packages/core/src/schema/common.ts +++ b/packages/core/src/schema/common.ts @@ -75,7 +75,7 @@ export type Column = | ManyColumn | EnumColumn; -export type NonVirtualColumn = Exclude; +export type MaterialColumn = Exclude; export type Table = { id: IdColumn } & { [columnName: string]: Column; diff --git a/packages/core/src/schema/utils.ts b/packages/core/src/schema/utils.ts index 8d83bd971..25c21cea5 100644 --- a/packages/core/src/schema/utils.ts +++ b/packages/core/src/schema/utils.ts @@ -5,6 +5,7 @@ import type { EnumColumn, JSONColumn, ManyColumn, + MaterialColumn, OneColumn, ReferenceColumn, ScalarColumn, @@ -46,6 +47,13 @@ export const isListColumn = (column: Column): boolean => { return column[" list"]; }; +/** Returns true if the column corresponds to a column in the database */ +export const isMaterialColumn = (column: Column): column is MaterialColumn => + isScalarColumn(column) || + isReferenceColumn(column) || + isEnumColumn(column) || + isJSONColumn(column); + export const isTable = ( tableOrEnum: Schema[string], ): tableOrEnum is { table: Table; constraints: Constraints } => From c349d415e61c95ce103e749a7be528a63a9df6cf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:13:28 -0400 Subject: [PATCH 033/122] chore: version packages (#947) Co-authored-by: github-actions[bot] --- .changeset/lovely-sheep-joke.md | 5 ----- packages/core/CHANGELOG.md | 6 ++++++ packages/core/package.json | 2 +- packages/create-ponder/CHANGELOG.md | 2 ++ packages/create-ponder/package.json | 2 +- packages/eslint-config-ponder/CHANGELOG.md | 2 ++ packages/eslint-config-ponder/package.json | 2 +- 7 files changed, 13 insertions(+), 8 deletions(-) delete mode 100644 .changeset/lovely-sheep-joke.md diff --git a/.changeset/lovely-sheep-joke.md b/.changeset/lovely-sheep-joke.md deleted file mode 100644 index 998bc8a5d..000000000 --- a/.changeset/lovely-sheep-joke.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@ponder/core": patch ---- - -Fixed an indexing bug causing the "syntax error at end of input" error. diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index b8036ce3c..b96268bdf 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,11 @@ # @ponder/core +## 0.4.40 + +### Patch Changes + +- [#946](https://github.com/ponder-sh/ponder/pull/946) [`d6cc62e69670cc683596ab0dc0b90357b7bce644`](https://github.com/ponder-sh/ponder/commit/d6cc62e69670cc683596ab0dc0b90357b7bce644) Thanks [@kyscott18](https://github.com/kyscott18)! - Fixed an indexing bug causing the "syntax error at end of input" error. + ## 0.4.39 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index dbdfec08d..586521390 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@ponder/core", - "version": "0.4.39", + "version": "0.4.40", "description": "An open-source framework for crypto application backends", "license": "MIT", "type": "module", diff --git a/packages/create-ponder/CHANGELOG.md b/packages/create-ponder/CHANGELOG.md index bae8fb176..728671e54 100644 --- a/packages/create-ponder/CHANGELOG.md +++ b/packages/create-ponder/CHANGELOG.md @@ -1,5 +1,7 @@ # create-ponder +## 0.4.40 + ## 0.4.39 ## 0.4.38 diff --git a/packages/create-ponder/package.json b/packages/create-ponder/package.json index 2e73370f8..5f604eb94 100644 --- a/packages/create-ponder/package.json +++ b/packages/create-ponder/package.json @@ -1,6 +1,6 @@ { "name": "create-ponder", - "version": "0.4.39", + "version": "0.4.40", "type": "module", "description": "A CLI tool to create Ponder apps", "license": "MIT", diff --git a/packages/eslint-config-ponder/CHANGELOG.md b/packages/eslint-config-ponder/CHANGELOG.md index 1e9e79e20..61adbd7b0 100644 --- a/packages/eslint-config-ponder/CHANGELOG.md +++ b/packages/eslint-config-ponder/CHANGELOG.md @@ -1,5 +1,7 @@ # eslint-config-ponder +## 0.4.40 + ## 0.4.39 ## 0.4.38 diff --git a/packages/eslint-config-ponder/package.json b/packages/eslint-config-ponder/package.json index 6da0951f9..1d23aeda3 100644 --- a/packages/eslint-config-ponder/package.json +++ b/packages/eslint-config-ponder/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-ponder", - "version": "0.4.39", + "version": "0.4.40", "description": "ESLint config for Ponder apps", "license": "MIT", "main": "./index.js", From 5c4bf59d6a9249a434cbf6191de1a5f1489a39a7 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 19 Jun 2024 16:49:47 -0400 Subject: [PATCH 034/122] server error handling --- packages/core/src/common/errors.ts | 2 +- packages/core/src/indexing/addStackTrace.ts | 12 +++-- packages/core/src/server/error.ts | 51 +++++++++++++++++++++ packages/core/src/server/service.ts | 16 ++++--- 4 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 packages/core/src/server/error.ts diff --git a/packages/core/src/common/errors.ts b/packages/core/src/common/errors.ts index c7689ba47..f0b13bc75 100644 --- a/packages/core/src/common/errors.ts +++ b/packages/core/src/common/errors.ts @@ -1,4 +1,4 @@ -class BaseError extends Error { +export class BaseError extends Error { override name = "BaseError"; meta: string[] = []; diff --git a/packages/core/src/indexing/addStackTrace.ts b/packages/core/src/indexing/addStackTrace.ts index af8ccb5c5..447f001f4 100644 --- a/packages/core/src/indexing/addStackTrace.ts +++ b/packages/core/src/indexing/addStackTrace.ts @@ -12,13 +12,17 @@ export const addStackTrace = (error: Error, options: Options) => { let userStackTrace: StackFrame[]; // Find first frame that occurred within user code. - const firstUserFrameIndex = stackTrace.findIndex((frame) => - frame.file?.includes(options.indexingDir), + const firstUserFrameIndex = stackTrace.findIndex( + (frame) => + frame.file?.includes(options.indexingDir) || + frame.file?.includes(options.serverDir), ); if (firstUserFrameIndex >= 0) { - userStackTrace = stackTrace.filter((frame) => - frame.file?.includes(options.indexingDir), + userStackTrace = stackTrace.filter( + (frame) => + frame.file?.includes(options.indexingDir) || + frame.file?.includes(options.serverDir), ); const firstUserFrame = stackTrace[firstUserFrameIndex]; diff --git a/packages/core/src/server/error.ts b/packages/core/src/server/error.ts new file mode 100644 index 000000000..eb0d2ffb7 --- /dev/null +++ b/packages/core/src/server/error.ts @@ -0,0 +1,51 @@ +import type { Common } from "@/common/common.js"; +import type { BaseError } from "@/common/errors.js"; +import { addStackTrace } from "@/indexing/addStackTrace.js"; +import { prettyPrint } from "@/utils/print.js"; +import type { Context, HonoRequest } from "hono"; +import { html } from "hono/html"; + +export const onError = async (_error: Error, c: Context, common: Common) => { + const error = _error as BaseError; + + addStackTrace(error, common.options); + + error.meta = Array.isArray(error.meta) ? error.meta : []; + error.meta.push( + `Request:\n${prettyPrint({ + path: c.req.path, + method: c.req.method, + body: await tryExtractRequestBody(c.req), + })}`, + ); + + common.logger.warn({ + service: "server", + msg: `An error occurred while handling a '${c.req.method}' request to the route '${c.req.path}'`, + error, + }); + + // 500: Internal Server Error + return c.text("", 500); +}; + +export const onNotFound = (c: Context) => { + return c.html( + html` +

Bad news!

+

The route "${c.req.path}" does not exist

`, + ); +}; + +const tryExtractRequestBody = async (request: HonoRequest) => { + // + try { + return await request.json(); + } catch { + try { + const text = await request.text(); + if (text !== "") return text; + } catch {} + } + return undefined; +}; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index aedccc14e..6796031f9 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -12,6 +12,7 @@ import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; import type { QueryResult } from "kysely"; +import { onError, onNotFound } from "./error.js"; type Server = { hono: Hono; @@ -160,13 +161,10 @@ export async function createServer({ for (const app of userApps) { for (const route of app.routes) { // Validate user routes don't conflict with ponder routes - if ( - route.path === "/_ponder/metrics" || - route.path === "/_ponder/health" - ) { + if (route.path.startsWith("/_ponder")) { common.logger.warn({ service: "server", - msg: `Ingoring path '${route.path}' with method '${route.method}' because '/_ponder' is reserved`, + msg: `Ingoring '${route.method}' handler for route '${route.path}' because '/_ponder' is reserved for internal use`, }); } @@ -195,10 +193,16 @@ export async function createServer({ if (userApps !== undefined) { for (const app of userApps) { - hono.route("/", app); + hono.route( + "/", + app.onError((error, c) => onError(error, c, common)), + ); } } + // Custom 404 error handling + hono.notFound(onNotFound); + // Create nodejs server let port = common.options.port; From aaa044de391c7459df9d26def0a41b56da4bc3f4 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 19 Jun 2024 18:24:04 -0400 Subject: [PATCH 035/122] return error stack in response --- packages/core/src/server/error.ts | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/core/src/server/error.ts b/packages/core/src/server/error.ts index eb0d2ffb7..f31ab116b 100644 --- a/packages/core/src/server/error.ts +++ b/packages/core/src/server/error.ts @@ -8,6 +8,21 @@ import { html } from "hono/html"; export const onError = async (_error: Error, c: Context, common: Common) => { const error = _error as BaseError; + // Find the filenames of the first three items of the stack + const regex = /(\S+\.(?:js|ts|mjs|cjs)):\d+:\d+/g; + const matches = error.stack?.match(regex); + const stack = matches + ?.map((m) => { + const path = m.trim(); + if (path.startsWith("(")) { + return path.slice(1); + } else if (path.startsWith("file://")) { + return path.slice(7); + } + return path; + }) + ?.slice(0, 3); + addStackTrace(error, common.options); error.meta = Array.isArray(error.meta) ? error.meta : []; @@ -26,7 +41,15 @@ export const onError = async (_error: Error, c: Context, common: Common) => { }); // 500: Internal Server Error - return c.text("", 500); + return c.json( + { + name: error.name, + message: error.message, + meta: error.meta, + stack, + }, + 500, + ); }; export const onNotFound = (c: Context) => { From cb12dc6ae286ab28e0840f9f2c9954f3d6409b5a Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 19 Jun 2024 18:31:40 -0400 Subject: [PATCH 036/122] text error --- packages/core/src/server/error.ts | 34 +++++++++++++------------------ 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/packages/core/src/server/error.ts b/packages/core/src/server/error.ts index f31ab116b..7e4d33867 100644 --- a/packages/core/src/server/error.ts +++ b/packages/core/src/server/error.ts @@ -8,20 +8,19 @@ import { html } from "hono/html"; export const onError = async (_error: Error, c: Context, common: Common) => { const error = _error as BaseError; - // Find the filenames of the first three items of the stack - const regex = /(\S+\.(?:js|ts|mjs|cjs)):\d+:\d+/g; + // Find the filename where the error occurred + const regex = /(\S+\.(?:js|ts|mjs|cjs)):\d+:\d+/; const matches = error.stack?.match(regex); - const stack = matches - ?.map((m) => { - const path = m.trim(); - if (path.startsWith("(")) { - return path.slice(1); - } else if (path.startsWith("file://")) { - return path.slice(7); - } - return path; - }) - ?.slice(0, 3); + const errorFile = (() => { + if (!matches?.[0]) return undefined; + const path = matches[0].trim(); + if (path.startsWith("(")) { + return path.slice(1); + } else if (path.startsWith("file://")) { + return path.slice(7); + } + return path; + })(); addStackTrace(error, common.options); @@ -41,13 +40,8 @@ export const onError = async (_error: Error, c: Context, common: Common) => { }); // 500: Internal Server Error - return c.json( - { - name: error.name, - message: error.message, - meta: error.meta, - stack, - }, + return c.text( + `${error.name}: ${error.message} occurred in '${errorFile}' while handling a '${c.req.method}' request to the route '${c.req.path}`, 500, ); }; From 0fae4bf7a93ff9c1a7f0e1a6204e3e440417b727 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 19 Jun 2024 18:51:19 -0400 Subject: [PATCH 037/122] . --- packages/core/src/server/error.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/server/error.ts b/packages/core/src/server/error.ts index 7e4d33867..4813f4385 100644 --- a/packages/core/src/server/error.ts +++ b/packages/core/src/server/error.ts @@ -41,7 +41,7 @@ export const onError = async (_error: Error, c: Context, common: Common) => { // 500: Internal Server Error return c.text( - `${error.name}: ${error.message} occurred in '${errorFile}' while handling a '${c.req.method}' request to the route '${c.req.path}`, + `${error.name}: ${error.message} occurred in '${errorFile}' while handling a '${c.req.method}' request to the route '${c.req.path}'`, 500, ); }; From 1a7762f2553f1d20c7045264d8b05ff2c1ef2a0b Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Thu, 20 Jun 2024 12:07:29 -0400 Subject: [PATCH 038/122] wip docs --- docs/pages/docs/_meta.ts | 6 +- docs/pages/docs/query/_meta.ts | 1 + docs/pages/docs/query/api-functions.mdx | 136 ++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 docs/pages/docs/query/api-functions.mdx diff --git a/docs/pages/docs/_meta.ts b/docs/pages/docs/_meta.ts index 5a7a0975b..a59037dab 100644 --- a/docs/pages/docs/_meta.ts +++ b/docs/pages/docs/_meta.ts @@ -30,11 +30,11 @@ export default { }, indexing: { display: "children", title: "Indexing" }, - "-- Query the database": { + "-- Query": { type: "separator", - title: "Query the database", + title: "Query", }, - query: { display: "children", title: "Query the database" }, + query: { display: "children", title: "Query" }, "-- Production": { type: "separator", diff --git a/docs/pages/docs/query/_meta.ts b/docs/pages/docs/query/_meta.ts index 19f84ab05..3e5b32faf 100644 --- a/docs/pages/docs/query/_meta.ts +++ b/docs/pages/docs/query/_meta.ts @@ -1,4 +1,5 @@ export default { + "api-functions": "API functions (experimental)", "graphql": "GraphQL", "direct-sql": "Direct SQL", }; diff --git a/docs/pages/docs/query/api-functions.mdx b/docs/pages/docs/query/api-functions.mdx new file mode 100644 index 000000000..514fc9547 --- /dev/null +++ b/docs/pages/docs/query/api-functions.mdx @@ -0,0 +1,136 @@ +--- +title: "API functions" +description: "Use API functions to customize the API layer of your app." +--- + +import { Callout, Steps } from "nextra/components"; + +# API functions (experimental) + +**API functions** are user-defined functions that respond to incoming Web requests. You can use them to customize the API layer of your app with direct SQL queries, authentication, data from external sources, and more. + +API functions are built on top of [Hono](https://hono.dev/), a fast and lightweight routing framework. + +## Get started + + + +### Install prerelease + +To use API functions, install the prerelease version of `@ponder/core`. + +```bash +npm install @ponder/core@alpha +``` + +### Create `src/api/index.ts` file + +To opt-in to API functions, create a file named `src/api/index.ts` with the following code. + +```ts filename="src/api/index.ts" +import { ponder } from "@/generated"; + +ponder.get("/hello", (c) => { + return c.text("Hello, world!"); +}); +``` + +### Send a request + +With the server running, visit `http://localhost:42069/hello` in your browser to see the response. + +### Add the GraphQL middleware + +To register the standard GraphQL API that was automatically included in versions `<0.5.0`, register the `graphql` middleware exported from `@ponder/core`. + +```ts filename="src/api/index.ts" {2,4} +import { ponder } from "@/generated"; +import { graphql } from "@ponder/core"; + +ponder.use("/graphql", graphql()); + +// ... +``` + + + +## Query the database + +API functions have read-only access to your database using a [Hono Variable](https://hono.dev/docs/api/context#set-get) named `db`. Unlike indexing functions, the `create`, `update`, `delete`, `createMany`, and `updateMany` methods are not available in API functions. + +### `findUnique` and `findMany` + +API functions can use `findUnique` and `findMany` to query the database. These methods work just like they do within indexing functions. [Read more](/docs/indexing/create-update-records#findmany) about the store API. + +```ts filename="src/api/index.ts" {5,7-9} +import { ponder } from "@/generated"; + +ponder.get("/account/:address", (c) => { + const address = c.req.param("address"); + const db = c.get("db"); + + const account = await db.Account.findUnique({ + id: address, + }); + + if (account) { + return c.json(account); + } else { + return c.status(404).json({ error: "Account not found" }); + } +}); +``` + +### Direct SQL + +Use `db.query(...){:ts}` to execute raw SQL queries against your database. + +```ts filename="src/api/index.ts" {5,7-9} +import { ponder } from "@/generated"; + +ponder.get("/:token/ticker", (c) => { + const token = c.req.param("token"); + const db = c.get("db"); + + const result = await db.query( + `SELECT ticker FROM "Token" WHERE id = ${token}` + ); + const ticker = result.rows[0]?.ticker; + + return c.text(ticker); +}); +``` + +## Register middleware + +Use `ponder.use(...){:ts}` to add middleware to your API functions. Middleware functions can modify the request and response objects, add logs, do authentication, and more. [Read more](https://hono.dev/docs/guides/middleware) about Hono middleware. + +```ts filename="src/api/index.ts" +import { ponder } from "@/generated"; + +ponder.use((c) => { + console.log("Request received:", c.req.url); + return c.next(); +}); +``` + +## Access the Hono instance + +You can register API functions and middleware using `ponder.get(...){:ts}`, `ponder.post(...){:ts}`, and `ponder.use(...){:ts}`. If you need to access the underlying Hono instance, use the `hono` property. + +```ts filename="src/api/index.ts" +import { ponder } from "@/generated"; + +ponder.hono.notFound((c) => { + return c.text("Custom 404 Message", 404); +}); + +// ... +``` + +## Internal routes + +Ponder registers internal routes under the `/_ponder/*` path. If you attempt to register API functions that conflict with internal routes, the build will fail. + +- `/_ponder/health`: Returns a `200` status code after the app has completed historical indexing OR the healthcheck timeout has expired, whichever comes first. [Read more](/docs/production/zero-downtime) about healthchecks. +- `/_ponder/metrics`: Returns Prometheus metrics. [Read more](/docs/advanced/metrics) about metrics. From 761bfe5c87cf83dfcb2055f3e5b4289c56f9bde9 Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Thu, 20 Jun 2024 12:07:39 -0400 Subject: [PATCH 039/122] fix biome config --- package.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index db36a190a..6eccf8d62 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,10 @@ "viem": "1.16.0" }, "lint-staged": { - "*.ts": ["biome format --no-errors-on-unmatched --write", "biome check"], + "*.ts": [ + "biome format --no-errors-on-unmatched --write", + "biome check --no-errors-on-unmatched" + ], "!(*.ts)": ["biome format --no-errors-on-unmatched --write"] }, "simple-git-hooks": { From 6254e26d809b9f09914578512a944f3fa5af6d1e Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 20 Jun 2024 12:38:05 -0400 Subject: [PATCH 040/122] file conventions --- .../src/{server => api}/index.ts | 4 + .../src/{indexing => }/index.ts | 0 packages/core/src/bin/commands/serve.ts | 4 +- packages/core/src/bin/utils/run.ts | 2 +- packages/core/src/build/service.ts | 98 ++++++------------- packages/core/src/common/options.ts | 8 +- packages/core/src/indexing/addStackTrace.ts | 12 +-- packages/core/src/server/service.ts | 56 ++++------- 8 files changed, 63 insertions(+), 121 deletions(-) rename examples/reference-erc20/src/{server => api}/index.ts (86%) rename examples/reference-erc20/src/{indexing => }/index.ts (100%) diff --git a/examples/reference-erc20/src/server/index.ts b/examples/reference-erc20/src/api/index.ts similarity index 86% rename from examples/reference-erc20/src/server/index.ts rename to examples/reference-erc20/src/api/index.ts index 074fe738d..cb2268e3e 100644 --- a/examples/reference-erc20/src/server/index.ts +++ b/examples/reference-erc20/src/api/index.ts @@ -1,12 +1,16 @@ import { ponder } from "@/generated"; import { graphql } from "@ponder/core"; +// write file ponder.use("/graphql", graphql()); ponder.get("/router", async (c) => { const db = c.get("db"); // await db.query(`UPDATE "Account" SET "isOwner" = 1`); + if (Math.random() > 0.5) { + throw new Error("kyle"); + } const account = await db.query<{ balance: bigint }>( `SELECT * FROM "Account" LIMIT 1`, diff --git a/examples/reference-erc20/src/indexing/index.ts b/examples/reference-erc20/src/index.ts similarity index 100% rename from examples/reference-erc20/src/indexing/index.ts rename to examples/reference-erc20/src/index.ts diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index 54d29c6a0..c76552813 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -67,7 +67,7 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { properties: { cli_command: "serve", ...buildPayload(initialResult.build) }, }); - const { databaseConfig, optionsConfig, schema, apps } = initialResult.build; + const { databaseConfig, optionsConfig, schema, app } = initialResult.build; common.options = { ...common.options, ...optionsConfig }; @@ -111,7 +111,7 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { }); const server = await createServer({ - apps, + app, readonlyStore, schema, query: (...query: any) => { diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index b562318da..9e2796cb4 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -104,7 +104,7 @@ export async function run({ }); const server = await createServer({ - apps: build.apps, + app: build.app, readonlyStore, schema, query: (...query: any) => { diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 3145a52c5..53c3f208f 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -31,8 +31,7 @@ const BUILD_ID_VERSION = "1"; export type Service = { // static common: Common; - indexingRegex: RegExp; - serverRegex: RegExp; + srcRegex: RegExp; // vite viteDevServer: ViteDevServer; @@ -54,7 +53,7 @@ export type Build = { // Indexing functions indexingFunctions: IndexingFunctions; // Server - apps?: Hono[]; + app?: Hono; }; export type BuildResult = @@ -68,7 +67,7 @@ type RawBuild = { indexingFunctions: RawIndexingFunctions; contentHash: string; }; - server: { apps?: Hono[] }; + server: { app?: Hono }; }; export const create = async ({ @@ -78,19 +77,12 @@ export const create = async ({ }): Promise => { const escapeRegex = /[.*+?^${}()|[\]\\]/g; - const escapedIndexingDir = common.options.indexingDir + const escapedSrcDir = common.options.srcDir // If on Windows, use a POSIX path for this regex. .replace(/\\/g, "/") // Escape special characters in the path. .replace(escapeRegex, "\\$&"); - const indexingRegex = new RegExp(`^${escapedIndexingDir}/.*\\.(ts|js)$`); - - const escapedServerDir = common.options.serverDir - // If on Windows, use a POSIX path for this regex. - .replace(/\\/g, "/") - // Escape special characters in the path. - .replace(escapeRegex, "\\$&"); - const serverRegex = new RegExp(`^${escapedServerDir}/.*\\.(ts|js)$`); + const srcRegex = new RegExp(`^${escapedSrcDir}/.*\\.(ts|js)$`); const viteLogger = { warnedMessages: new Set(), @@ -142,8 +134,7 @@ export const create = async ({ return { common, - indexingRegex, - serverRegex, + srcRegex, viteDevServer, viteNodeServer, viteNodeRunner, @@ -244,21 +235,13 @@ export const start = async ( const hasSchemaUpdate = invalidated.includes( common.options.schemaFile.replace(/\\/g, "/"), ); - const hasIndexingFunctionUpdate = invalidated.some((file) => - buildService.indexingRegex.test(file), - ); - const hasServerUpdate = invalidated.includes( - common.options.serverDir.replace(/\\/g, "/"), + const hasSrcUpdate = invalidated.some((file) => + buildService.srcRegex.test(file), ); // This branch could trigger if you change a `note.txt` file within `src/`. // Note: We could probably do a better job filtering out files in `isFileIgnored`. - if ( - !hasConfigUpdate && - !hasSchemaUpdate && - !hasIndexingFunctionUpdate && - !hasServerUpdate - ) { + if (!hasConfigUpdate && !hasSchemaUpdate && !hasSrcUpdate) { return; } @@ -287,7 +270,7 @@ export const start = async ( rawBuild.schema = result; } - if (hasIndexingFunctionUpdate) { + if (hasSrcUpdate) { const result = await executeIndexingFunctions(buildService); if (result.status === "error") { onBuild({ status: "error", error: result.error }); @@ -296,7 +279,7 @@ export const start = async ( rawBuild.indexingFunctions = result; } - if (hasServerUpdate) { + if (hasSrcUpdate) { const result = await executeServer(buildService); if (result.status === "error") { onBuild({ status: "error", error: result.error }); @@ -392,8 +375,9 @@ const executeIndexingFunctions = async ( | { status: "error"; error: Error } > => { const pattern = path - .join(buildService.common.options.indexingDir, "**/*.{js,mjs,ts,mts}") + .join(buildService.common.options.srcDir, "**/*.{js,mjs,ts,mts}") .replace(/\\/g, "/"); + // TODO(kyle) ignore server file const files = glob.sync(pattern); const executeResults = await Promise.all( @@ -447,56 +431,34 @@ const executeServer = async ( ): Promise< | { status: "success"; - apps?: Hono[]; + app?: Hono; } | { status: "error"; error: Error } > => { - const doesServerExist = fs.existsSync(buildService.common.options.serverDir); + const doesServerExist = fs.existsSync(buildService.common.options.serverFile); if (doesServerExist === false) { return { status: "success" }; } - const pattern = path - .join(buildService.common.options.serverDir, "**/*.{js,mjs,ts,mts}") - .replace(/\\/g, "/"); - const files = glob.sync(pattern); - - if (files.length === 0) { - return { status: "success" }; - } - - const executeResults = await Promise.all( - files.map(async (file) => ({ - ...(await executeFile(buildService, { file })), - file, - })), - ); - - const apps: Hono[] = []; - - for (const executeResult of executeResults) { - if (executeResult.status === "error") { - buildService.common.logger.error({ - service: "build", - msg: `Error while executing '${path.relative( - buildService.common.options.rootDir, - executeResult.file, - )}':`, - error: executeResult.error, - }); - - return executeResult; - } + const executeResult = await executeFile(buildService, { + file: buildService.common.options.serverFile, + }); - if (executeResult.exports?.ponder?.hono === undefined) { - continue; - } + if (executeResult.status === "error") { + buildService.common.logger.error({ + service: "build", + msg: `Error while executing '${path.relative( + buildService.common.options.rootDir, + buildService.common.options.serverFile, + )}':`, + error: executeResult.error, + }); - apps.push(executeResult.exports.ponder.hono as Hono); + return executeResult; } - return { status: "success", apps }; + return { status: "success", app: executeResult.exports?.ponder?.hono }; }; const validateAndBuild = async ( @@ -569,7 +531,7 @@ const validateAndBuild = async ( graphQLSchema, indexingFunctions: buildConfigAndIndexingFunctionsResult.indexingFunctions, - apps: rawBuild.server.apps, + app: rawBuild.server.app, }, }; }; diff --git a/packages/core/src/common/options.ts b/packages/core/src/common/options.ts index ed95747e8..5cd63d945 100644 --- a/packages/core/src/common/options.ts +++ b/packages/core/src/common/options.ts @@ -8,9 +8,9 @@ export type Options = { configFile: string; schemaFile: string; + serverFile: string; rootDir: string; - indexingDir: string; - serverDir: string; + srcDir: string; generatedDir: string; ponderDir: string; logDir: string; @@ -79,8 +79,8 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => { rootDir, configFile: path.join(rootDir, cliOptions.config), schemaFile: path.join(rootDir, "ponder.schema.ts"), - indexingDir: path.join(rootDir, "src", "indexing"), - serverDir: path.join(rootDir, "src", "server"), + serverFile: path.join(rootDir, "src", "api", "index.ts"), + srcDir: path.join(rootDir, "src"), generatedDir: path.join(rootDir, "generated"), ponderDir: path.join(rootDir, ".ponder"), logDir: path.join(rootDir, ".ponder", "logs"), diff --git a/packages/core/src/indexing/addStackTrace.ts b/packages/core/src/indexing/addStackTrace.ts index 447f001f4..fd10a80d9 100644 --- a/packages/core/src/indexing/addStackTrace.ts +++ b/packages/core/src/indexing/addStackTrace.ts @@ -12,17 +12,13 @@ export const addStackTrace = (error: Error, options: Options) => { let userStackTrace: StackFrame[]; // Find first frame that occurred within user code. - const firstUserFrameIndex = stackTrace.findIndex( - (frame) => - frame.file?.includes(options.indexingDir) || - frame.file?.includes(options.serverDir), + const firstUserFrameIndex = stackTrace.findIndex((frame) => + frame.file?.includes(options.srcDir), ); if (firstUserFrameIndex >= 0) { - userStackTrace = stackTrace.filter( - (frame) => - frame.file?.includes(options.indexingDir) || - frame.file?.includes(options.serverDir), + userStackTrace = stackTrace.filter((frame) => + frame.file?.includes(options.srcDir), ); const firstUserFrame = stackTrace[firstUserFrameIndex]; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 6796031f9..38b30221d 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -22,13 +22,13 @@ type Server = { }; export async function createServer({ - apps: userApps, + app: userApp, schema, readonlyStore, query, common, }: { - apps?: Hono[]; + app?: Hono; schema: Schema; readonlyStore: ReadonlyStore; query: (query: string) => Promise>; @@ -66,8 +66,7 @@ export async function createServer({ } return c.text("Historical indexing is not complete.", 503); - }) - .get("/ready", async (c) => c.text("", 200)); + }); const metricsMiddleware = createMiddleware(async (c, next) => { const commonLabels = { method: c.req.method, path: c.req.path }; @@ -155,49 +154,30 @@ export async function createServer({ .route("/_ponder", ponderApp) .use(contextMiddleware); - if (userApps !== undefined) { - const routes = new Set(); - - for (const app of userApps) { - for (const route of app.routes) { - // Validate user routes don't conflict with ponder routes - if (route.path.startsWith("/_ponder")) { - common.logger.warn({ - service: "server", - msg: `Ingoring '${route.method}' handler for route '${route.path}' because '/_ponder' is reserved for internal use`, - }); - } - - // Validate user routes don't conflict have duplicates - const _route = `${route.method}_${route.path}`; - if (routes.has(_route)) { - common.logger.warn({ - service: "server", - msg: `Path '${route.path}' with method '${route.method}' already has been defined`, - }); - } else { - routes.add(_route); - } + if (userApp !== undefined) { + for (const route of userApp.routes) { + // Validate user routes don't conflict with ponder routes + if (route.path.startsWith("/_ponder")) { + common.logger.warn({ + service: "server", + msg: `Ingoring '${route.method}' handler for route '${route.path}' because '/_ponder' is reserved for internal use`, + }); } - - hono.route("/", app); } common.logger.debug({ service: "server", - msg: `Detected a custom server with routes: [${userApps - .flatMap((app) => app.routes.map((r) => r.path)) + msg: `Detected a custom server with routes: [${userApp.routes + .map((r) => r.path) .join(", ")}]`, }); } - if (userApps !== undefined) { - for (const app of userApps) { - hono.route( - "/", - app.onError((error, c) => onError(error, c, common)), - ); - } + if (userApp !== undefined) { + hono.route( + "/", + userApp.onError((error, c) => onError(error, c, common)), + ); } // Custom 404 error handling From b1591824f8a7adff8ee6f141796f23c5be2bc22b Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 20 Jun 2024 13:05:18 -0400 Subject: [PATCH 041/122] sql tagged template --- examples/reference-erc20/src/api/index.ts | 12 +++++++----- packages/core/src/bin/commands/serve.ts | 8 +++----- packages/core/src/bin/utils/run.ts | 10 +++++----- packages/core/src/index.ts | 1 + packages/core/src/server/service.ts | 4 ++-- packages/core/src/types/virtual.ts | 7 ++----- 6 files changed, 20 insertions(+), 22 deletions(-) diff --git a/examples/reference-erc20/src/api/index.ts b/examples/reference-erc20/src/api/index.ts index cb2268e3e..dc777ee4f 100644 --- a/examples/reference-erc20/src/api/index.ts +++ b/examples/reference-erc20/src/api/index.ts @@ -1,5 +1,5 @@ import { ponder } from "@/generated"; -import { graphql } from "@ponder/core"; +import { graphql, sql } from "@ponder/core"; // write file ponder.use("/graphql", graphql()); @@ -8,12 +8,14 @@ ponder.get("/router", async (c) => { const db = c.get("db"); // await db.query(`UPDATE "Account" SET "isOwner" = 1`); - if (Math.random() > 0.5) { - throw new Error("kyle"); - } + // if (Math.random() > 0.5) { + // throw new Error("kyle"); + // } + + const v = 1; const account = await db.query<{ balance: bigint }>( - `SELECT * FROM "Account" LIMIT 1`, + sql`SELECT * FROM "Account" LIMIT ${v}`, ); if (account.rows.length === 0) { diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index c76552813..0ec09197b 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -8,7 +8,7 @@ import { PostgresDatabaseService } from "@/database/postgres/service.js"; import type { NamespaceInfo } from "@/database/service.js"; import { getReadonlyStore } from "@/indexing-store/readonly.js"; import { createServer } from "@/server/service.js"; -import { sql } from "kysely"; +import { type RawBuilder, sql } from "kysely"; import type { CliOptions } from "../ponder.js"; import { setupShutdown } from "../utils/shutdown.js"; @@ -114,10 +114,8 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { app, readonlyStore, schema, - query: (...query: any) => { - return sql - .raw(query) - .execute(database.readonlyDb.withSchema(userNamespace)); + query: (query: RawBuilder) => { + return query.execute(database.readonlyDb.withSchema(userNamespace)); }, common, }); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 9e2796cb4..221d05a87 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -23,7 +23,7 @@ import { } from "@/utils/checkpoint.js"; import { never } from "@/utils/never.js"; import { createQueue } from "@ponder/common"; -import { sql } from "kysely"; +import type { RawBuilder } from "kysely"; export type RealtimeEvent = | { @@ -107,10 +107,10 @@ export async function run({ app: build.app, readonlyStore, schema, - query: (...query: any) => { - return sql - .raw(query) - .execute(database.readonlyDb.withSchema(namespaceInfo.userNamespace)); + query: (query: RawBuilder) => { + return query.execute( + database.readonlyDb.withSchema(namespaceInfo.userNamespace), + ); }, common, }); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 114521a36..900bf1a07 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -20,3 +20,4 @@ export type BlockConfig = Prettify; export type DatabaseConfig = Prettify; export { graphql } from "@/graphql/index.js"; +export { sql } from "kysely"; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 38b30221d..8bf5e478b 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -11,7 +11,7 @@ import { Hono } from "hono"; import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; -import type { QueryResult } from "kysely"; +import type { QueryResult, RawBuilder } from "kysely"; import { onError, onNotFound } from "./error.js"; type Server = { @@ -31,7 +31,7 @@ export async function createServer({ app?: Hono; schema: Schema; readonlyStore: ReadonlyStore; - query: (query: string) => Promise>; + query: (query: RawBuilder) => Promise>; common: Common; }): Promise { // Create hono app diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index 0e9784eac..c9945d182 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -18,7 +18,7 @@ import type { } from "@/types/eth.js"; import type { DatabaseModel } from "@/types/model.js"; import type { Hono } from "hono"; -import type { QueryResult } from "kysely"; +import type { QueryResult, RawBuilder } from "kysely"; import type { Prettify } from "./utils.js"; export namespace Virtual { @@ -168,10 +168,7 @@ export namespace Virtual { > >; } & { - query: ( - sqlFragment: string, - ...parameters: unknown[] - ) => Promise>; + query: (query: RawBuilder) => Promise>; }; export type Context< From 47e3ada9aa163e7af3f71b22eb4d9e15824a6613 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 20 Jun 2024 13:07:27 -0400 Subject: [PATCH 042/122] remove custom not found --- packages/core/src/server/service.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 8bf5e478b..6f47d0c14 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -12,7 +12,7 @@ import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; import type { QueryResult, RawBuilder } from "kysely"; -import { onError, onNotFound } from "./error.js"; +import { onError } from "./error.js"; type Server = { hono: Hono; @@ -180,9 +180,6 @@ export async function createServer({ ); } - // Custom 404 error handling - hono.notFound(onNotFound); - // Create nodejs server let port = common.options.port; From 5905e57d98c0fd3cff3b6b549061b1258ecbe6e4 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 20 Jun 2024 13:14:48 -0400 Subject: [PATCH 043/122] fix e2e tests --- packages/core/src/_test/e2e/erc20/src/{server => api}/index.ts | 0 packages/core/src/_test/e2e/erc20/src/{indexing => }/index.ts | 0 .../core/src/_test/e2e/factory/src/{server => api}/index.ts | 0 packages/core/src/_test/e2e/factory/src/{indexing => }/index.ts | 0 packages/core/src/bin/commands/serve.ts | 2 +- 5 files changed, 1 insertion(+), 1 deletion(-) rename packages/core/src/_test/e2e/erc20/src/{server => api}/index.ts (100%) rename packages/core/src/_test/e2e/erc20/src/{indexing => }/index.ts (100%) rename packages/core/src/_test/e2e/factory/src/{server => api}/index.ts (100%) rename packages/core/src/_test/e2e/factory/src/{indexing => }/index.ts (100%) diff --git a/packages/core/src/_test/e2e/erc20/src/server/index.ts b/packages/core/src/_test/e2e/erc20/src/api/index.ts similarity index 100% rename from packages/core/src/_test/e2e/erc20/src/server/index.ts rename to packages/core/src/_test/e2e/erc20/src/api/index.ts diff --git a/packages/core/src/_test/e2e/erc20/src/indexing/index.ts b/packages/core/src/_test/e2e/erc20/src/index.ts similarity index 100% rename from packages/core/src/_test/e2e/erc20/src/indexing/index.ts rename to packages/core/src/_test/e2e/erc20/src/index.ts diff --git a/packages/core/src/_test/e2e/factory/src/server/index.ts b/packages/core/src/_test/e2e/factory/src/api/index.ts similarity index 100% rename from packages/core/src/_test/e2e/factory/src/server/index.ts rename to packages/core/src/_test/e2e/factory/src/api/index.ts diff --git a/packages/core/src/_test/e2e/factory/src/indexing/index.ts b/packages/core/src/_test/e2e/factory/src/index.ts similarity index 100% rename from packages/core/src/_test/e2e/factory/src/indexing/index.ts rename to packages/core/src/_test/e2e/factory/src/index.ts diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index 0ec09197b..d09f62ff9 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -8,7 +8,7 @@ import { PostgresDatabaseService } from "@/database/postgres/service.js"; import type { NamespaceInfo } from "@/database/service.js"; import { getReadonlyStore } from "@/indexing-store/readonly.js"; import { createServer } from "@/server/service.js"; -import { type RawBuilder, sql } from "kysely"; +import type { RawBuilder } from "kysely"; import type { CliOptions } from "../ponder.js"; import { setupShutdown } from "../utils/shutdown.js"; From 979cb9b7d50cd1787060cf346dfd6cb0d6385308 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 20 Jun 2024 13:22:35 -0400 Subject: [PATCH 044/122] . --- packages/core/src/_test/e2e/erc20/src/index.ts | 4 ++-- packages/core/src/_test/e2e/factory/src/index.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/src/_test/e2e/erc20/src/index.ts b/packages/core/src/_test/e2e/erc20/src/index.ts index 27cd8f8b1..029728b8a 100644 --- a/packages/core/src/_test/e2e/erc20/src/index.ts +++ b/packages/core/src/_test/e2e/erc20/src/index.ts @@ -3,8 +3,8 @@ import { ponder } from "@/generated"; // biome-ignore lint/suspicious/noRedeclare: :) declare const ponder: import("@/index.js").Virtual.Registry< - typeof import("../../ponder.config.js").default, - typeof import("../../ponder.schema.js").default + typeof import("../ponder.config.js").default, + typeof import("../ponder.schema.js").default >; ponder.on( diff --git a/packages/core/src/_test/e2e/factory/src/index.ts b/packages/core/src/_test/e2e/factory/src/index.ts index 9123e6144..15e36a2e6 100644 --- a/packages/core/src/_test/e2e/factory/src/index.ts +++ b/packages/core/src/_test/e2e/factory/src/index.ts @@ -3,8 +3,8 @@ import { ponder } from "@/generated"; // biome-ignore lint/suspicious/noRedeclare: :) declare const ponder: import("@/index.js").Virtual.Registry< - typeof import("../../ponder.config.js").default, - typeof import("../../ponder.schema.js").default + typeof import("../ponder.config.js").default, + typeof import("../ponder.schema.js").default >; ponder.on("Pair:Swap", async ({ event, context }) => { From 3e60f95acc57c2200fe51c48234a5ee646ee836e Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:03:24 -0400 Subject: [PATCH 045/122] docs tweak --- docs/pages/docs/query/api-functions.mdx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/pages/docs/query/api-functions.mdx b/docs/pages/docs/query/api-functions.mdx index 514fc9547..a884ee888 100644 --- a/docs/pages/docs/query/api-functions.mdx +++ b/docs/pages/docs/query/api-functions.mdx @@ -15,12 +15,12 @@ API functions are built on top of [Hono](https://hono.dev/), a fast and lightwei -### Install prerelease +### Install `hono` and `@ponder/core@next` -To use API functions, install the prerelease version of `@ponder/core`. +To use API functions, install `hono` and the prerelease version of `@ponder/core`. ```bash -npm install @ponder/core@alpha +npm install hono @ponder/core@next ``` ### Create `src/api/index.ts` file @@ -87,13 +87,14 @@ Use `db.query(...){:ts}` to execute raw SQL queries against your database. ```ts filename="src/api/index.ts" {5,7-9} import { ponder } from "@/generated"; +import { sql } from "@ponder/core"; ponder.get("/:token/ticker", (c) => { const token = c.req.param("token"); const db = c.get("db"); const result = await db.query( - `SELECT ticker FROM "Token" WHERE id = ${token}` + sql`SELECT ticker FROM "Token" WHERE id = ${token}` ); const ticker = result.rows[0]?.ticker; From f018a69fc6540d074dfaf570cfac535768246183 Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:11:30 -0400 Subject: [PATCH 046/122] chore: wip changeset --- .changeset/shy-donuts-battle.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/shy-donuts-battle.md diff --git a/.changeset/shy-donuts-battle.md b/.changeset/shy-donuts-battle.md new file mode 100644 index 000000000..6cbd50291 --- /dev/null +++ b/.changeset/shy-donuts-battle.md @@ -0,0 +1,6 @@ +--- +"create-ponder": minor +"@ponder/core": minor +--- + +[Experimental] Introduced API functions. [Read more](https://ponder-docs-git-kjs-api2-ponder-sh.vercel.app/docs/query/api-functions). From a171ab49b4f5fda5bdb1508de1393ec43ad0fb50 Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:43:31 -0400 Subject: [PATCH 047/122] fix: docs --- docs/pages/docs/query/api-functions.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/pages/docs/query/api-functions.mdx b/docs/pages/docs/query/api-functions.mdx index a884ee888..9fca08c2d 100644 --- a/docs/pages/docs/query/api-functions.mdx +++ b/docs/pages/docs/query/api-functions.mdx @@ -85,7 +85,7 @@ ponder.get("/account/:address", (c) => { Use `db.query(...){:ts}` to execute raw SQL queries against your database. -```ts filename="src/api/index.ts" {5,7-9} +```ts filename="src/api/index.ts" {6,8-10} import { ponder } from "@/generated"; import { sql } from "@ponder/core"; @@ -106,7 +106,7 @@ ponder.get("/:token/ticker", (c) => { Use `ponder.use(...){:ts}` to add middleware to your API functions. Middleware functions can modify the request and response objects, add logs, do authentication, and more. [Read more](https://hono.dev/docs/guides/middleware) about Hono middleware. -```ts filename="src/api/index.ts" +```ts filename="src/api/index.ts" {3} import { ponder } from "@/generated"; ponder.use((c) => { @@ -119,7 +119,7 @@ ponder.use((c) => { You can register API functions and middleware using `ponder.get(...){:ts}`, `ponder.post(...){:ts}`, and `ponder.use(...){:ts}`. If you need to access the underlying Hono instance, use the `hono` property. -```ts filename="src/api/index.ts" +```ts filename="src/api/index.ts" {3} import { ponder } from "@/generated"; ponder.hono.notFound((c) => { From 4e05dfc5b41ed416a7861438115030a02657474b Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 24 Jun 2024 17:20:08 -0400 Subject: [PATCH 048/122] start drizzle --- packages/core/package.json | 1 + packages/core/src/database/sqlite/service.ts | 5 +- packages/core/src/drizzle/db.ts | 15 + packages/core/src/drizzle/index.ts | 3 + packages/core/src/drizzle/runtime.test.ts | 52 +++ packages/core/src/drizzle/runtime.ts | 79 +++++ packages/core/src/drizzle/select.ts | 344 +++++++++++++++++++ packages/core/src/drizzle/table.test.ts | 28 ++ packages/core/src/drizzle/table.ts | 59 ++++ packages/core/src/drizzle/virtual.ts | 32 ++ pnpm-lock.yaml | 132 +++++-- 11 files changed, 712 insertions(+), 38 deletions(-) create mode 100644 packages/core/src/drizzle/db.ts create mode 100644 packages/core/src/drizzle/index.ts create mode 100644 packages/core/src/drizzle/runtime.test.ts create mode 100644 packages/core/src/drizzle/runtime.ts create mode 100644 packages/core/src/drizzle/select.ts create mode 100644 packages/core/src/drizzle/table.test.ts create mode 100644 packages/core/src/drizzle/table.ts create mode 100644 packages/core/src/drizzle/virtual.ts diff --git a/packages/core/package.json b/packages/core/package.json index 586521390..f30b9c101 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -87,6 +87,7 @@ "@types/react": "^18.2.38", "@viem/anvil": "^0.0.6", "@wagmi/cli": "^1.5.2", + "drizzle-orm": "^0.31.2", "execa": "^8.0.1", "rimraf": "^5.0.5", "tsup": "^8.0.1", diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index 058e0945a..e644652c9 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -51,7 +51,8 @@ export class SqliteDatabaseService implements BaseDatabaseService { private userNamespace: string; private internalNamespace: string; - private internalDatabase: SqliteDatabase; + internalDatabase: SqliteDatabase; + userDatabase: SqliteDatabase; private syncDatabase: SqliteDatabase; db: HeadlessKysely; @@ -91,6 +92,8 @@ export class SqliteDatabaseService implements BaseDatabaseService { `ATTACH DATABASE '${userDatabaseFile}' AS ${this.userNamespace}`, ); + this.userDatabase = createSqliteDatabase(userDatabaseFile); + this.db = new HeadlessKysely({ name: "internal", common, diff --git a/packages/core/src/drizzle/db.ts b/packages/core/src/drizzle/db.ts new file mode 100644 index 000000000..99edbe375 --- /dev/null +++ b/packages/core/src/drizzle/db.ts @@ -0,0 +1,15 @@ +import type { Column, SQLWrapper, SelectedFields, Table } from "drizzle-orm"; +import type { SelectBuilder } from "./select.js"; + +export type DrizzleDb = { + select(): SelectBuilder; + select>( + fields: TSelection, + ): SelectBuilder; + select( + fields?: SelectedFields, + ): SelectBuilder | undefined, "async", void>; + execute: >( + query: SQLWrapper, + ) => record[]; +}; diff --git a/packages/core/src/drizzle/index.ts b/packages/core/src/drizzle/index.ts new file mode 100644 index 000000000..4fa6fc6c0 --- /dev/null +++ b/packages/core/src/drizzle/index.ts @@ -0,0 +1,3 @@ +export type { DrizzleDb } from "./db.js"; +export { createDrizzleDb, convertToDrizzleTable } from "./runtime.js"; +export type { ConvertToDrizzleTable } from "./table.js"; diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts new file mode 100644 index 000000000..a205b1aa9 --- /dev/null +++ b/packages/core/src/drizzle/runtime.test.ts @@ -0,0 +1,52 @@ +import { + setupCommon, + setupDatabaseServices, + setupIsolatedDatabase, +} from "@/_test/setup.js"; +import { SqliteDatabaseService } from "@/database/sqlite/service.js"; +import type { HistoricalStore } from "@/indexing-store/store.js"; +import { createSchema } from "@/schema/schema.js"; +import { beforeEach, expect, test } from "vitest"; +import type { DrizzleDb } from "./db.js"; +import { convertToDrizzleTable, createDb } from "./runtime.js"; + +beforeEach(setupCommon); +beforeEach(setupIsolatedDatabase); + +test("runtime select", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.string(), + }), + })); + + const { database, cleanup, indexingStore } = await setupDatabaseServices( + context, + { + schema, + }, + ); + + let db: DrizzleDb; + + if (database instanceof SqliteDatabaseService) { + db = createDb({ + kind: "sqlite", + database: database.userDatabase, + }) as any; + } + + await indexingStore.create({ tableName: "table", id: "kyle" }); + + await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); + + const rows = await db! + .select() + .from(convertToDrizzleTable("table", schema.table.table, "sqlite")); + + expect(rows).toHaveLength(1); + + expect(rows[0]).toMatchObject({ id: "kyle" }); + + await cleanup(); +}); diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts new file mode 100644 index 000000000..7c8728518 --- /dev/null +++ b/packages/core/src/drizzle/runtime.ts @@ -0,0 +1,79 @@ +import type { Table as PonderTable } from "@/schema/common.js"; +import { + isMaterialColumn, + isOptionalColumn, + isReferenceColumn, + isScalarColumn, +} from "@/schema/utils.js"; +import type { SqliteDatabase } from "@/utils/sqlite.js"; +import type { Table } from "drizzle-orm"; +import { drizzle as drizzleSQLite } from "drizzle-orm/better-sqlite3"; +import { drizzle as drizzlePg } from "drizzle-orm/node-postgres"; +import { pgTable } from "drizzle-orm/pg-core"; +import { + integer as SQLiteInteger, + text as SQLiteText, + sqliteTable, +} from "drizzle-orm/sqlite-core"; +import type { Pool } from "pg"; + +export const createDrizzleDb = ( + database: + | { kind: "postgres"; pool: Pool } + | { kind: "sqlite"; database: SqliteDatabase }, +) => { + if (database.kind === "postgres") { + const drizzle = drizzlePg(database.pool); + drizzle.execute; + return { + // @ts-ignore + select: (...args: any[]) => drizzle.select(...args), + execute: (query: any) => drizzle.execute(query), + }; + } else { + const drizzle = drizzleSQLite(database.database); + return { + // @ts-ignore + select: (...args: any[]) => drizzle.select(...args), + execute: (query: any) => drizzle.all(query), + }; + } +}; + +export const convertToDrizzleTable = ( + tableName: string, + table: PonderTable, + kind: "sqlite" | "postgres", +): Table => { + const columns = Object.entries(table).reduce<{ [columnName: string]: any }>( + (acc, [columnName, column]) => { + if (isMaterialColumn(column)) { + if (isScalarColumn(column) || isReferenceColumn(column)) { + let drizzleColumn = + column[" scalar"] === "string" + ? SQLiteText(columnName) + : column[" scalar"] === "int" + ? SQLiteInteger(columnName) + : undefined; + + // apply column constraints + if (columnName === "id") { + drizzleColumn = drizzleColumn!.primaryKey(); + } else if (isOptionalColumn(column) === false) { + drizzleColumn = drizzleColumn!.notNull(); + } + + acc[columnName] = drizzleColumn; + } + } + return acc; + }, + {}, + ); + + if (kind === "postgres") { + return pgTable(tableName, columns); + } else { + return sqliteTable(tableName, columns); + } +}; diff --git a/packages/core/src/drizzle/select.ts b/packages/core/src/drizzle/select.ts new file mode 100644 index 000000000..620f2f0c8 --- /dev/null +++ b/packages/core/src/drizzle/select.ts @@ -0,0 +1,344 @@ +import type { + Assume, + Column, + MakeColumnConfig, + QueryPromise, + SelectedFieldsOrdered as SelectFieldsOrderedBase, + SelectedFields, + Subquery, + Table, + UpdateTableConfig, + entityKind, +} from "drizzle-orm"; +import type { TypedQueryBuilder } from "drizzle-orm/query-builders/query-builder"; +import type { + BuildSubquerySelection, + GetSelectTableName, + GetSelectTableSelection, + JoinNullability, + JoinType, + SelectMode, + SelectResult, + SetOperator, +} from "drizzle-orm/query-builders/select.types"; +import type { ColumnsSelection, Placeholder, SQL, View } from "drizzle-orm/sql"; +import type { TableWithColumns, ViewWithSelection } from "./table.js"; + +export type SelectBuilder< + TSelection extends SelectedFields | undefined, + TResultType extends "sync" | "async", + TRunResult, + TBuilderMode extends "db" | "qb" = "db", +> = { + from: ( + source: TFrom, + ) => CreateSelectFromBuilderMode< + TBuilderMode, + GetSelectTableName, + TResultType, + TRunResult, + TSelection extends undefined ? GetSelectTableSelection : TSelection, + TSelection extends undefined ? "single" : "partial" + >; +}; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L31 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L30 + */ +export type SelectJoinConfig = { + on: SQL; + table: Table | Subquery | View | SQL; + alias: string | undefined; + joinType: JoinType; +}; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/query-builders/select.types.ts#L75 + */ +type MapColumnsToTableAlias< + TColumns extends ColumnsSelection, + TAlias extends string, +> = { + [Key in keyof TColumns]: TColumns[Key] extends Column + ? Column["_"], TAlias>> + : TColumns[Key]; +} & {}; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L38 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L30 + */ +export type BuildAliasTable< + tableOrView extends Table | View, + alias extends string, +> = tableOrView extends Table + ? TableWithColumns< + UpdateTableConfig< + tableOrView["_"]["config"], + { + name: alias; + columns: MapColumnsToTableAlias; + } + > + > + : tableOrView extends View + ? ViewWithSelection< + alias, + tableOrView["_"]["existing"], + MapColumnsToTableAlias + > + : never; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L52 + */ +export type SelectConfig = { + withList?: Subquery[]; + fields: Record; + fieldsFlat?: SelectedFieldsOrdered; + where?: SQL; + having?: SQL; + table: Table | Subquery | View | SQL; + limit?: number | Placeholder; + offset?: number | Placeholder; + joins?: SelectJoinConfig[]; + orderBy?: (Column | SQL | SQL.Aliased)[]; + groupBy?: (Column | SQL | SQL.Aliased)[]; + distinct?: boolean; + setOperators: { + rightSelect: TypedQueryBuilder; + type: SetOperator; + isAll: boolean; + orderBy?: (Column | SQL | SQL.Aliased)[]; + limit?: number | Placeholder; + offset?: number | Placeholder; + }[]; +}; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L122 + */ +export type SelectedFieldsOrdered = SelectFieldsOrderedBase; + +export type SelectHKTBase = { + tableName: string | undefined; + resultType: "sync" | "async"; + runResult: unknown; + selection: unknown; + selectMode: SelectMode; + nullabilityMap: unknown; + dynamic: boolean; + excludedMethods: string; + result: unknown; + selectedFields: unknown; + _type: unknown; +}; + +export interface SelectHKT extends SelectHKTBase { + _type: SelectBase< + this["tableName"], + this["resultType"], + this["runResult"], + Assume, + this["selectMode"], + Assume>, + this["dynamic"], + this["excludedMethods"], + Assume, + Assume + >; +} + +export type SelectKind< + T extends SelectHKTBase, + TTableName extends string | undefined, + TResultType extends "sync" | "async", + TRunResult, + TSelection extends ColumnsSelection, + TSelectMode extends SelectMode, + TNullabilityMap extends Record, + TDynamic extends boolean, + TExcludedMethods extends string, + TResult = SelectResult[], + TSelectedFields = BuildSubquerySelection, +> = (T & { + tableName: TTableName; + resultType: TResultType; + runResult: TRunResult; + selection: TSelection; + selectMode: TSelectMode; + nullabilityMap: TNullabilityMap; + dynamic: TDynamic; + excludedMethods: TExcludedMethods; + result: TResult; + selectedFields: TSelectedFields; +})["_type"]; + +export interface SelectQueryBuilderHKT extends SelectHKTBase { + _type: SelectQueryBuilderBase< + SelectQueryBuilderHKT, + this["tableName"], + this["resultType"], + this["runResult"], + Assume, + this["selectMode"], + Assume>, + this["dynamic"], + this["excludedMethods"], + Assume, + Assume + >; +} + +/** + * Partial implementation of the select query builder + * + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.ts#L126 + */ +export type SelectQueryBuilderBase< + THKT extends SelectHKTBase, + TTableName extends string | undefined, + TResultType extends "sync" | "async", + TRunResult, + TSelection extends ColumnsSelection, + TSelectMode extends SelectMode, + TNullabilityMap extends Record< + string, + JoinNullability + > = TTableName extends string ? Record : {}, + TDynamic extends boolean = false, + TExcludedMethods extends string = never, + TResult extends any[] = SelectResult< + TSelection, + TSelectMode, + TNullabilityMap + >[], + TSelectedFields extends ColumnsSelection = BuildSubquerySelection< + TSelection, + TNullabilityMap + >, +> = TypedQueryBuilder & { + [entityKind]: string; + _: { + readonly hkt: THKT; + readonly tableName: TTableName; + readonly resultType: TResultType; + readonly runResult: TRunResult; + readonly selection: TSelection; + readonly selectMode: TSelectMode; + readonly nullabilityMap: TNullabilityMap; + readonly dynamic: TDynamic; + readonly excludedMethods: TExcludedMethods; + readonly result: TResult; + readonly selectedFields: TSelectedFields; + }; + + where: ( + where: ((aliases: TSelection) => SQL | undefined) | SQL | undefined, + ) => SelectWithout< + SelectQueryBuilderBase< + THKT, + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode, + TNullabilityMap, + TDynamic, + TExcludedMethods, + TResult, + TSelectedFields + >, + TDynamic, + "where" + >; +}; + +export type CreateSelectFromBuilderMode< + TBuilderMode extends "db" | "qb", + TTableName extends string | undefined, + TResultType extends "sync" | "async", + TRunResult, + TSelection extends ColumnsSelection, + TSelectMode extends SelectMode, +> = TBuilderMode extends "db" + ? SelectBase + : SelectQueryBuilderBase< + SelectQueryBuilderHKT, + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode + >; + +export type AnySelectQueryBuilder = SelectQueryBuilderBase< + any, + any, + any, + any, + any, + any, + any, + any, + any, + any, + any +>; + +type SelectWithout< + T extends AnySelectQueryBuilder, + TDynamic extends boolean, + K extends keyof T & string, + TResetExcluded extends boolean = false, +> = TDynamic extends true + ? T + : Omit< + SelectKind< + T["_"]["hkt"], + T["_"]["tableName"], + T["_"]["resultType"], + T["_"]["runResult"], + T["_"]["selection"], + T["_"]["selectMode"], + T["_"]["nullabilityMap"], + TDynamic, + TResetExcluded extends true ? K : T["_"]["excludedMethods"] | K, + T["_"]["result"], + T["_"]["selectedFields"] + >, + TResetExcluded extends true ? K : T["_"]["excludedMethods"] | K + >; + +export type SelectBase< + TTableName extends string | undefined, + TResultType extends "sync" | "async", + TRunResult, + TSelection extends ColumnsSelection, + TSelectMode extends SelectMode = "single", + TNullabilityMap extends Record< + string, + JoinNullability + > = TTableName extends string ? Record : {}, + TDynamic extends boolean = false, + TExcludedMethods extends string = never, + TResult = SelectResult[], + TSelectedFields extends ColumnsSelection = BuildSubquerySelection< + TSelection, + TNullabilityMap + >, +> = SelectQueryBuilderBase< + SelectHKT, + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode, + TNullabilityMap, + TDynamic, + TExcludedMethods, + TResult, + TSelectedFields +> & + QueryPromise; diff --git a/packages/core/src/drizzle/table.test.ts b/packages/core/src/drizzle/table.test.ts new file mode 100644 index 000000000..181ad00c3 --- /dev/null +++ b/packages/core/src/drizzle/table.test.ts @@ -0,0 +1,28 @@ +import type { IdColumn, ScalarColumn } from "@/schema/common.js"; +import { expectTypeOf, test } from "vitest"; +import type { DrizzleDb } from "./db.js"; +import type { ConvertToDrizzleTable } from "./table.js"; + +test("select query promise", async () => { + const table = {} as ConvertToDrizzleTable< + "table", + { id: IdColumn<"string">; name: ScalarColumn<"int", true> } + >; + + const result = await ({} as DrizzleDb).select({ id: table.id }).from(table); + // ^? + + expectTypeOf<{ id: string }[]>(result); +}); + +test("select optional column", async () => { + const table = {} as ConvertToDrizzleTable< + "table", + { id: IdColumn<"string">; n: ScalarColumn<"int", true> } + >; + + const result = await ({} as DrizzleDb).select().from(table); + // ^? + + expectTypeOf<{ id: string; n: number | null }[]>(result); +}); diff --git a/packages/core/src/drizzle/table.ts b/packages/core/src/drizzle/table.ts new file mode 100644 index 000000000..8fd52074a --- /dev/null +++ b/packages/core/src/drizzle/table.ts @@ -0,0 +1,59 @@ +import type { + ExtractNonVirtualColumnNames, + Table as PonderTable, +} from "@/schema/common.js"; +import type { InferScalarType } from "@/schema/infer.js"; +import type { + BuildColumns, + ColumnBuilderBase, + ColumnsSelection, + Table, + TableConfig, + View, +} from "drizzle-orm"; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/table.ts#L49 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/table.ts#L43 + */ +export type TableWithColumns = Table & { + [key in keyof T["columns"]]: T["columns"][key]; +}; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/view.ts#L154 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/view.ts#L305 + */ +export type ViewWithSelection< + TName extends string, + TExisting extends boolean, + TSelection extends ColumnsSelection, +> = View & TSelection; + +/** + * @returns TableWithColumns + */ +export type ConvertToDrizzleTable< + tableName extends string, + table extends PonderTable, +> = TableWithColumns<{ + name: tableName; + schema: undefined; + columns: BuildColumns< + tableName, + { + [columnName in ExtractNonVirtualColumnNames]: ColumnBuilderBase<{ + name: columnName & string; + dataType: "string"; + columnType: "custom"; + data: InferScalarType; + driverParam: unknown; + enumValues: undefined; + notNull: table[columnName][" optional"] extends true ? false : true; + primaryKey: columnName extends "id" ? true : false; + }>; + }, + "common" + >; + dialect: "common"; +}>; diff --git a/packages/core/src/drizzle/virtual.ts b/packages/core/src/drizzle/virtual.ts new file mode 100644 index 000000000..fa668802a --- /dev/null +++ b/packages/core/src/drizzle/virtual.ts @@ -0,0 +1,32 @@ +export { + sql, + eq, + gt, + gte, + lt, + lte, + ne, + isNull, + isNotNull, + inArray, + notInArray, + exists, + notExists, + between, + notBetween, + like, + notIlike, + not, + asc, + desc, + and, + or, + count, + countDistinct, + avg, + avgDistinct, + sum, + sumDistinct, + max, + min, +} from "drizzle-orm"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a7a753e83..b8deada45 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -690,6 +690,9 @@ importers: '@wagmi/cli': specifier: ^1.5.2 version: 1.5.2(typescript@5.0.4) + drizzle-orm: + specifier: ^0.31.2 + version: 0.31.2(@types/better-sqlite3@7.6.10)(@types/pg@8.10.9)(@types/react@18.2.46)(better-sqlite3@10.0.0)(kysely@0.26.3)(pg@8.11.3)(react@18.2.0) execa: specifier: ^8.0.1 version: 8.0.1 @@ -5274,7 +5277,6 @@ packages: dependencies: bindings: 1.5.0 prebuild-install: 7.1.2 - dev: false /binary-extensions@2.2.0: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} @@ -5305,7 +5307,6 @@ packages: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} dependencies: file-uri-to-path: 1.0.0 - dev: false /bintrees@1.0.2: resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} @@ -5324,7 +5325,6 @@ packages: buffer: 5.7.1 inherits: 2.0.4 readable-stream: 3.6.2 - dev: false /bl@5.1.0: resolution: {integrity: sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==} @@ -5453,7 +5453,6 @@ packages: /buffer-writer@2.0.0: resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==} engines: {node: '>=4'} - dev: false /buffer-xor@1.0.3: resolution: {integrity: sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==} @@ -5464,7 +5463,6 @@ packages: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 - dev: false /buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} @@ -6539,7 +6537,6 @@ packages: engines: {node: '>=10'} dependencies: mimic-response: 3.1.0 - dev: false /dedent@0.7.0: resolution: {integrity: sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==} @@ -6555,7 +6552,6 @@ packages: /deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} - dev: false /deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -6616,7 +6612,6 @@ packages: /detect-libc@2.0.3: resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==} engines: {node: '>=8'} - dev: false /detect-package-manager@2.0.1: resolution: {integrity: sha512-j/lJHyoLlWi6G1LDdLgvUtz60Zo5GEj+sVYtTVXnYLDPuzgC3llMxonXym9zIwhhUII8vjdw0LXxavpLqTbl1A==} @@ -6780,6 +6775,98 @@ packages: engines: {node: '>=10'} dev: true + /drizzle-orm@0.31.2(@types/better-sqlite3@7.6.10)(@types/pg@8.10.9)(@types/react@18.2.46)(better-sqlite3@10.0.0)(kysely@0.26.3)(pg@8.11.3)(react@18.2.0): + resolution: {integrity: sha512-QnenevbnnAzmbNzQwbhklvIYrDE8YER8K7kSrAWQSV1YvFCdSQPzj+jzqRdTSsV2cDqSpQ0NXGyL1G9I43LDLg==} + peerDependencies: + '@aws-sdk/client-rds-data': '>=3' + '@cloudflare/workers-types': '>=3' + '@electric-sql/pglite': '>=0.1.1' + '@libsql/client': '*' + '@neondatabase/serverless': '>=0.1' + '@op-engineering/op-sqlite': '>=2' + '@opentelemetry/api': ^1.4.1 + '@planetscale/database': '>=1' + '@tidbcloud/serverless': '*' + '@types/better-sqlite3': '*' + '@types/pg': '*' + '@types/react': '>=18' + '@types/sql.js': '*' + '@vercel/postgres': '>=0.8.0' + '@xata.io/client': '*' + better-sqlite3: '>=7' + bun-types: '*' + expo-sqlite: '>=13.2.0' + knex: '*' + kysely: '*' + mysql2: '>=2' + pg: '>=8' + postgres: '>=3' + react: '>=18' + sql.js: '>=1' + sqlite3: '>=5' + peerDependenciesMeta: + '@aws-sdk/client-rds-data': + optional: true + '@cloudflare/workers-types': + optional: true + '@electric-sql/pglite': + optional: true + '@libsql/client': + optional: true + '@neondatabase/serverless': + optional: true + '@op-engineering/op-sqlite': + optional: true + '@opentelemetry/api': + optional: true + '@planetscale/database': + optional: true + '@tidbcloud/serverless': + optional: true + '@types/better-sqlite3': + optional: true + '@types/pg': + optional: true + '@types/react': + optional: true + '@types/sql.js': + optional: true + '@vercel/postgres': + optional: true + '@xata.io/client': + optional: true + better-sqlite3: + optional: true + bun-types: + optional: true + expo-sqlite: + optional: true + knex: + optional: true + kysely: + optional: true + mysql2: + optional: true + pg: + optional: true + postgres: + optional: true + react: + optional: true + sql.js: + optional: true + sqlite3: + optional: true + dependencies: + '@types/better-sqlite3': 7.6.10 + '@types/pg': 8.10.9 + '@types/react': 18.2.46 + better-sqlite3: 10.0.0 + kysely: 0.26.3 + pg: 8.11.3 + react: 18.2.0 + dev: true + /dset@3.1.3: resolution: {integrity: sha512-20TuZZHCEZ2O71q9/+8BwKwZ0QtD9D8ObhrihJPr+vLLYlSuAU3/zL4cSlgbfeoGHTjCSJBa7NGcrF9/Bx/WJQ==} engines: {node: '>=4'} @@ -7585,7 +7672,6 @@ packages: /expand-template@2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} - dev: false /extend-shallow@2.0.1: resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} @@ -7710,7 +7796,6 @@ packages: /file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} - dev: false /filelist@1.0.4: resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==} @@ -8020,7 +8105,6 @@ packages: /github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} - dev: false /github-slugger@2.0.0: resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} @@ -8680,7 +8764,6 @@ packages: /ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - dev: false /ink@4.4.1(@types/react@18.2.46)(react@18.2.0): resolution: {integrity: sha512-rXckvqPBB0Krifk5rn/5LvQGmyXwCUpBfmTwbkQNBY9JY8RSl3b8OftBNEYxg4+SWUhEKcPifgope28uL9inlA==} @@ -9513,7 +9596,6 @@ packages: /kysely@0.26.3: resolution: {integrity: sha512-yWSgGi9bY13b/W06DD2OCDDHQmq1kwTGYlQ4wpZkMOJqMGCstVCFIvxCCVG4KfY1/3G0MhDAcZsip/Lw8/vJWw==} engines: {node: '>=14.0.0'} - dev: false /language-subtag-registry@0.3.22: resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==} @@ -10781,7 +10863,6 @@ packages: /mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} - dev: false /min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} @@ -10874,7 +10955,6 @@ packages: /mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} - dev: false /mkdirp@0.5.6: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} @@ -10953,7 +11033,6 @@ packages: /napi-build-utils@1.0.2: resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} - dev: false /native-abort-controller@1.0.4(abort-controller@3.0.0): resolution: {integrity: sha512-zp8yev7nxczDJMoP6pDxyD20IU0T22eX8VwN2ztDccKvSZhRaV33yP1BGwKSZfXuqWUzsXopVFjBdau9OOAwMQ==} @@ -11115,7 +11194,6 @@ packages: engines: {node: '>=10'} dependencies: semver: 7.6.2 - dev: false /node-addon-api@2.0.2: resolution: {integrity: sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==} @@ -11526,7 +11604,6 @@ packages: /packet-reader@1.0.0: resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} - dev: false /param-case@3.0.4: resolution: {integrity: sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==} @@ -11684,12 +11761,10 @@ packages: /pg-cloudflare@1.1.1: resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==} requiresBuild: true - dev: false optional: true /pg-connection-string@2.6.2: resolution: {integrity: sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==} - dev: false /pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} @@ -11706,7 +11781,6 @@ packages: pg: '>=8.0' dependencies: pg: 8.11.3 - dev: false /pg-protocol@1.6.0: resolution: {integrity: sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==} @@ -11720,7 +11794,6 @@ packages: postgres-bytea: 1.0.0 postgres-date: 1.0.7 postgres-interval: 1.2.0 - dev: false /pg-types@4.0.1: resolution: {integrity: sha512-hRCSDuLII9/LE3smys1hRHcu5QGcLs9ggT7I/TCs0IE+2Eesxi9+9RWAAwZ0yaGjxoWICF/YHLOEjydGujoJ+g==} @@ -11753,13 +11826,11 @@ packages: pgpass: 1.0.5 optionalDependencies: pg-cloudflare: 1.1.1 - dev: false /pgpass@1.0.5: resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} dependencies: split2: 4.2.0 - dev: false /picocolors@1.0.0: resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} @@ -11910,7 +11981,6 @@ packages: /postgres-array@2.0.0: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} - dev: false /postgres-array@3.0.2: resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} @@ -11920,7 +11990,6 @@ packages: /postgres-bytea@1.0.0: resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} engines: {node: '>=0.10.0'} - dev: false /postgres-bytea@3.0.0: resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} @@ -11932,7 +12001,6 @@ packages: /postgres-date@1.0.7: resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} engines: {node: '>=0.10.0'} - dev: false /postgres-date@2.0.1: resolution: {integrity: sha512-YtMKdsDt5Ojv1wQRvUhnyDJNSr2dGIC96mQVKz7xufp07nfuFONzdaowrMHjlAzY6GDLd4f+LUHHAAM1h4MdUw==} @@ -11944,7 +12012,6 @@ packages: engines: {node: '>=0.10.0'} dependencies: xtend: 4.0.2 - dev: false /postgres-interval@3.0.0: resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} @@ -11982,7 +12049,6 @@ packages: simple-get: 4.0.1 tar-fs: 2.1.1 tunnel-agent: 0.6.0 - dev: false /preferred-pm@3.1.2: resolution: {integrity: sha512-nk7dKrcW8hfCZ4H6klWcdRknBOXWzNQByJ0oJyX97BOupsYD+FzLS4hflgEu/uPUEHZCuRfMxzCBsuWd7OzT8Q==} @@ -12116,7 +12182,6 @@ packages: dependencies: end-of-stream: 1.4.4 once: 1.4.0 - dev: false /punycode@1.4.1: resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} @@ -12175,7 +12240,6 @@ packages: ini: 1.3.8 minimist: 1.2.8 strip-json-comments: 2.0.1 - dev: false /react-countup@6.5.0(react@18.2.0): resolution: {integrity: sha512-26JFHbUHsHxu8SetkJwWVIUEkaNnrj4P9msxNGC8tS4hGr1bngRzbwtJYOgXD2G/ItjaKJ3JfYKd85sw7qRVeA==} @@ -12896,7 +12960,6 @@ packages: /simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} - dev: false /simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} @@ -12904,7 +12967,6 @@ packages: decompress-response: 6.0.0 once: 1.4.0 simple-concat: 1.0.1 - dev: false /simple-git-hooks@2.9.0: resolution: {integrity: sha512-waSQ5paUQtyGC0ZxlHmcMmD9I1rRXauikBwX31bX58l5vTOhCEcBC5Bi+ZDkPXTjDnZAF8TbCqKBY+9+sVPScw==} @@ -13072,7 +13134,6 @@ packages: /split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} - dev: false /sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} @@ -13296,7 +13357,6 @@ packages: /strip-json-comments@2.0.1: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} - dev: false /strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} @@ -13489,7 +13549,6 @@ packages: mkdirp-classic: 0.5.3 pump: 3.0.0 tar-stream: 2.2.0 - dev: false /tar-stream@1.6.2: resolution: {integrity: sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==} @@ -13513,7 +13572,6 @@ packages: fs-constants: 1.0.0 inherits: 2.0.4 readable-stream: 3.6.2 - dev: false /tar@6.2.0: resolution: {integrity: sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==} From 791a1eb4b71f8a6aa58f1f3526c33ac4cd8eaeb0 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 24 Jun 2024 19:40:06 -0400 Subject: [PATCH 049/122] select types --- examples/reference-erc20/src/api/index.ts | 20 ++++++------------- packages/core/src/common/codegen.ts | 15 ++++++++++++++ packages/core/src/drizzle/select.ts | 1 + packages/core/src/index.ts | 3 ++- packages/core/src/types/virtual.ts | 24 +++++------------------ 5 files changed, 29 insertions(+), 34 deletions(-) diff --git a/examples/reference-erc20/src/api/index.ts b/examples/reference-erc20/src/api/index.ts index dc777ee4f..e12447519 100644 --- a/examples/reference-erc20/src/api/index.ts +++ b/examples/reference-erc20/src/api/index.ts @@ -1,5 +1,6 @@ -import { ponder } from "@/generated"; -import { graphql, sql } from "@ponder/core"; +import { Account } from "ponder:db"; +import { ponder } from "ponder:virtual"; +import { graphql } from "@ponder/core"; // write file ponder.use("/graphql", graphql()); @@ -7,20 +8,11 @@ ponder.use("/graphql", graphql()); ponder.get("/router", async (c) => { const db = c.get("db"); - // await db.query(`UPDATE "Account" SET "isOwner" = 1`); - // if (Math.random() > 0.5) { - // throw new Error("kyle"); - // } + const account = await db.select().from(Account); - const v = 1; - - const account = await db.query<{ balance: bigint }>( - sql`SELECT * FROM "Account" LIMIT ${v}`, - ); - - if (account.rows.length === 0) { + if (account.length === 0) { return c.text("Not Found!"); } else { - return c.text(`Balance: ${account.rows[0]!.balance.toString()}`); + return c.text(`Balance: ${account[0]!.balance.toString()}`); } }); diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index 6fe66b47a..17205342a 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -30,6 +30,21 @@ declare module "@/generated" { Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; } + +declare module "ponder:db" { + import type { ConvertToDrizzleTable } from "@ponder/core"; + + type schema = typeof import("./ponder.schema.ts").default; + + const drizzleTables: { + [tableName in keyof schema]: ConvertToDrizzleTable< + tableName, + schema[tableName]["table"] + >; + }; + + export = drizzleTables; +} `; export function runCodegen({ diff --git a/packages/core/src/drizzle/select.ts b/packages/core/src/drizzle/select.ts index 620f2f0c8..b1e7776dc 100644 --- a/packages/core/src/drizzle/select.ts +++ b/packages/core/src/drizzle/select.ts @@ -338,6 +338,7 @@ export type SelectBase< TNullabilityMap, TDynamic, TExcludedMethods, + // @ts-ignore TResult, TSelectedFields > & diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 900bf1a07..97d02bcac 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -20,4 +20,5 @@ export type BlockConfig = Prettify; export type DatabaseConfig = Prettify; export { graphql } from "@/graphql/index.js"; -export { sql } from "kysely"; + +export type { ConvertToDrizzleTable } from "@/drizzle/index.js"; diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index c9945d182..7b5978e80 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -6,6 +6,7 @@ import type { SafeEventNames, SafeFunctionNames, } from "@/config/utilityTypes.js"; +import type { DrizzleDb } from "@/drizzle/db.js"; import type { ReadOnlyClient } from "@/indexing/ponderActions.js"; import type { Schema as BuilderSchema } from "@/schema/common.js"; import type { InferSchemaType } from "@/schema/infer.js"; @@ -18,7 +19,6 @@ import type { } from "@/types/eth.js"; import type { DatabaseModel } from "@/types/model.js"; import type { Hono } from "hono"; -import type { QueryResult, RawBuilder } from "kysely"; import type { Prettify } from "./utils.js"; export namespace Virtual { @@ -157,20 +157,6 @@ export namespace Virtual { >[property], > = ([base] extends [never] ? undefined : base) | override; - type ReadonlyDb = { - [key in keyof InferSchemaType]: Prettify< - Pick< - DatabaseModel< - // @ts-ignore - InferSchemaType[key] - >, - "findUnique" | "findMany" - > - >; - } & { - query: (query: RawBuilder) => Promise>; - }; - export type Context< config extends Config, schema extends BuilderSchema, @@ -262,9 +248,9 @@ export namespace Virtual { }, ) => Promise | void, ) => void; - get: Hono<{ Variables: { db: ReadonlyDb } }>["get"]; - post: Hono<{ Variables: { db: ReadonlyDb } }>["post"]; - use: Hono<{ Variables: { db: ReadonlyDb } }>["use"]; - hono: Hono<{ Variables: { db: ReadonlyDb } }>; + get: Hono<{ Variables: { db: DrizzleDb } }>["get"]; + post: Hono<{ Variables: { db: DrizzleDb } }>["post"]; + use: Hono<{ Variables: { db: DrizzleDb } }>["use"]; + hono: Hono<{ Variables: { db: DrizzleDb } }>; }; } From 9484a1395be59d7ee2843061cb75de3f019caaf8 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 24 Jun 2024 20:48:37 -0400 Subject: [PATCH 050/122] db select injection --- packages/core/package.json | 2 +- packages/core/src/bin/utils/run.ts | 11 ++-- packages/core/src/build/plugin.ts | 15 +++++ packages/core/src/drizzle/runtime.test.ts | 4 +- pnpm-lock.yaml | 73 +++++++++++++++++------ 5 files changed, 79 insertions(+), 26 deletions(-) diff --git a/packages/core/package.json b/packages/core/package.json index f30b9c101..3073f515c 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -57,6 +57,7 @@ "dataloader": "^2.2.2", "detect-package-manager": "^3.0.1", "dotenv": "^16.3.1", + "drizzle-orm": "^0.31.2", "emittery": "^1.0.1", "ethereum-bloom-filters": "^1.0.10", "glob": "^10.3.10", @@ -87,7 +88,6 @@ "@types/react": "^18.2.38", "@viem/anvil": "^0.0.6", "@wagmi/cli": "^1.5.2", - "drizzle-orm": "^0.31.2", "execa": "^8.0.1", "rimraf": "^5.0.5", "tsup": "^8.0.1", diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 221d05a87..73ca9814a 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -23,7 +23,6 @@ import { } from "@/utils/checkpoint.js"; import { never } from "@/utils/never.js"; import { createQueue } from "@ponder/common"; -import type { RawBuilder } from "kysely"; export type RealtimeEvent = | { @@ -107,11 +106,11 @@ export async function run({ app: build.app, readonlyStore, schema, - query: (query: RawBuilder) => { - return query.execute( - database.readonlyDb.withSchema(namespaceInfo.userNamespace), - ); - }, + // @ts-ignore + database: + database.kind === "sqlite" + ? { kind: "sqlite", database: database.userDatabase } + : { kind: "postgres" }, common, }); diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 5669ce4d3..75abdf6fe 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -31,6 +31,21 @@ export function replaceStateless(code: string, regex: RegExp, shim: string) { export const vitePluginPonder = (): Plugin => { return { name: "ponder", + load: (id) => { + if (id === "ponder:db") { + return `import schema from "ponder.schema"; +import { convertToDrizzleTable } from "@ponder/core"; +let drizzleTables = Object.fromEntries( + Object.entries(schema).map(([tableName, table]) => [ + tableName, + convertToDrizzleTable(tableName, table.table, "sqlite"), + ]), +); +module.exports = drizzleTables; +`; + } + return null; + }, transform: (code, id) => { if (ponderRegex.test(code)) { const s = replaceStateless(code, ponderRegex, shim); diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index a205b1aa9..ed1a1168d 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -8,7 +8,7 @@ import type { HistoricalStore } from "@/indexing-store/store.js"; import { createSchema } from "@/schema/schema.js"; import { beforeEach, expect, test } from "vitest"; import type { DrizzleDb } from "./db.js"; -import { convertToDrizzleTable, createDb } from "./runtime.js"; +import { convertToDrizzleTable, createDrizzleDb } from "./runtime.js"; beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); @@ -30,7 +30,7 @@ test("runtime select", async (context) => { let db: DrizzleDb; if (database instanceof SqliteDatabaseService) { - db = createDb({ + db = createDrizzleDb({ kind: "sqlite", database: database.userDatabase, }) as any; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b8deada45..63ba35649 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -605,6 +605,9 @@ importers: dotenv: specifier: ^16.3.1 version: 16.3.1 + drizzle-orm: + specifier: ^0.31.2 + version: 0.31.2(@types/better-sqlite3@7.6.10)(@types/pg@8.10.9)(@types/react@18.2.46)(better-sqlite3@10.0.0)(kysely@0.26.3)(pg@8.11.3)(react@18.2.0) emittery: specifier: ^1.0.1 version: 1.0.1 @@ -690,9 +693,6 @@ importers: '@wagmi/cli': specifier: ^1.5.2 version: 1.5.2(typescript@5.0.4) - drizzle-orm: - specifier: ^0.31.2 - version: 0.31.2(@types/better-sqlite3@7.6.10)(@types/pg@8.10.9)(@types/react@18.2.46)(better-sqlite3@10.0.0)(kysely@0.26.3)(pg@8.11.3)(react@18.2.0) execa: specifier: ^8.0.1 version: 8.0.1 @@ -2876,7 +2876,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dependencies: ajv: 6.12.6 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.0 @@ -3220,7 +3220,7 @@ packages: deprecated: Use @eslint/config-array instead dependencies: '@humanwhocodes/object-schema': 2.0.1 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -4225,7 +4225,6 @@ packages: resolution: {integrity: sha512-TZBjD+yOsyrUJGmcUj6OS3JADk3+UZcNv3NOBqGkM09bZdi28fNZw8ODqbMOLfKCu7RYCO62/ldq1iHbzxqoPw==} dependencies: '@types/node': 20.11.24 - dev: true /@types/bn.js@5.1.5: resolution: {integrity: sha512-V46N0zwKRF5Q00AZ6hWtN0T8gGmDUaUzLWQvHFo5yThtVwK/VCenFY3wXVbOvNfajEpsTfQM4IN9k/d6gUVX3A==} @@ -4399,7 +4398,6 @@ packages: '@types/node': 20.11.24 pg-protocol: 1.6.0 pg-types: 4.0.1 - dev: true /@types/prompts@2.4.9: resolution: {integrity: sha512-qTxFi6Buiu8+50/+3DGIWLHM6QuWsEKugJnnP6iv2Mc4ncxE4A/OJkjuVOA+5X0X1S/nq5VJRa8Lu+nwcvbrKA==} @@ -5277,6 +5275,7 @@ packages: dependencies: bindings: 1.5.0 prebuild-install: 7.1.2 + dev: false /binary-extensions@2.2.0: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} @@ -5307,6 +5306,7 @@ packages: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} dependencies: file-uri-to-path: 1.0.0 + dev: false /bintrees@1.0.2: resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} @@ -5325,6 +5325,7 @@ packages: buffer: 5.7.1 inherits: 2.0.4 readable-stream: 3.6.2 + dev: false /bl@5.1.0: resolution: {integrity: sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==} @@ -5453,6 +5454,7 @@ packages: /buffer-writer@2.0.0: resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==} engines: {node: '>=4'} + dev: false /buffer-xor@1.0.3: resolution: {integrity: sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==} @@ -5463,6 +5465,7 @@ packages: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 + dev: false /buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} @@ -6501,6 +6504,18 @@ packages: ms: 2.1.3 dev: true + /debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + dev: true + /debug@4.3.4(supports-color@8.1.1): resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -6537,6 +6552,7 @@ packages: engines: {node: '>=10'} dependencies: mimic-response: 3.1.0 + dev: false /dedent@0.7.0: resolution: {integrity: sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==} @@ -6552,6 +6568,7 @@ packages: /deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} + dev: false /deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -6612,6 +6629,7 @@ packages: /detect-libc@2.0.3: resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==} engines: {node: '>=8'} + dev: false /detect-package-manager@2.0.1: resolution: {integrity: sha512-j/lJHyoLlWi6G1LDdLgvUtz60Zo5GEj+sVYtTVXnYLDPuzgC3llMxonXym9zIwhhUII8vjdw0LXxavpLqTbl1A==} @@ -6865,7 +6883,7 @@ packages: kysely: 0.26.3 pg: 8.11.3 react: 18.2.0 - dev: true + dev: false /dset@3.1.3: resolution: {integrity: sha512-20TuZZHCEZ2O71q9/+8BwKwZ0QtD9D8ObhrihJPr+vLLYlSuAU3/zL4cSlgbfeoGHTjCSJBa7NGcrF9/Bx/WJQ==} @@ -7385,7 +7403,7 @@ packages: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.4(supports-color@8.1.1) + debug: 4.3.4 doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -7672,6 +7690,7 @@ packages: /expand-template@2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} + dev: false /extend-shallow@2.0.1: resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} @@ -7796,6 +7815,7 @@ packages: /file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + dev: false /filelist@1.0.4: resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==} @@ -8105,6 +8125,7 @@ packages: /github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + dev: false /github-slugger@2.0.0: resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} @@ -8764,6 +8785,7 @@ packages: /ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + dev: false /ink@4.4.1(@types/react@18.2.46)(react@18.2.0): resolution: {integrity: sha512-rXckvqPBB0Krifk5rn/5LvQGmyXwCUpBfmTwbkQNBY9JY8RSl3b8OftBNEYxg4+SWUhEKcPifgope28uL9inlA==} @@ -9596,6 +9618,7 @@ packages: /kysely@0.26.3: resolution: {integrity: sha512-yWSgGi9bY13b/W06DD2OCDDHQmq1kwTGYlQ4wpZkMOJqMGCstVCFIvxCCVG4KfY1/3G0MhDAcZsip/Lw8/vJWw==} engines: {node: '>=14.0.0'} + dev: false /language-subtag-registry@0.3.22: resolution: {integrity: sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==} @@ -10863,6 +10886,7 @@ packages: /mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} + dev: false /min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} @@ -10955,6 +10979,7 @@ packages: /mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + dev: false /mkdirp@0.5.6: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} @@ -11033,6 +11058,7 @@ packages: /napi-build-utils@1.0.2: resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} + dev: false /native-abort-controller@1.0.4(abort-controller@3.0.0): resolution: {integrity: sha512-zp8yev7nxczDJMoP6pDxyD20IU0T22eX8VwN2ztDccKvSZhRaV33yP1BGwKSZfXuqWUzsXopVFjBdau9OOAwMQ==} @@ -11194,6 +11220,7 @@ packages: engines: {node: '>=10'} dependencies: semver: 7.6.2 + dev: false /node-addon-api@2.0.2: resolution: {integrity: sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==} @@ -11374,7 +11401,6 @@ packages: /obuf@1.1.2: resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} - dev: true /on-exit-leak-free@2.1.2: resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} @@ -11604,6 +11630,7 @@ packages: /packet-reader@1.0.0: resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} + dev: false /param-case@3.0.4: resolution: {integrity: sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==} @@ -11761,10 +11788,12 @@ packages: /pg-cloudflare@1.1.1: resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==} requiresBuild: true + dev: false optional: true /pg-connection-string@2.6.2: resolution: {integrity: sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==} + dev: false /pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} @@ -11773,7 +11802,6 @@ packages: /pg-numeric@1.0.2: resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} engines: {node: '>=4'} - dev: true /pg-pool@3.6.1(pg@8.11.3): resolution: {integrity: sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==} @@ -11781,6 +11809,7 @@ packages: pg: '>=8.0' dependencies: pg: 8.11.3 + dev: false /pg-protocol@1.6.0: resolution: {integrity: sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==} @@ -11794,6 +11823,7 @@ packages: postgres-bytea: 1.0.0 postgres-date: 1.0.7 postgres-interval: 1.2.0 + dev: false /pg-types@4.0.1: resolution: {integrity: sha512-hRCSDuLII9/LE3smys1hRHcu5QGcLs9ggT7I/TCs0IE+2Eesxi9+9RWAAwZ0yaGjxoWICF/YHLOEjydGujoJ+g==} @@ -11806,7 +11836,6 @@ packages: postgres-date: 2.0.1 postgres-interval: 3.0.0 postgres-range: 1.1.3 - dev: true /pg@8.11.3: resolution: {integrity: sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==} @@ -11826,11 +11855,13 @@ packages: pgpass: 1.0.5 optionalDependencies: pg-cloudflare: 1.1.1 + dev: false /pgpass@1.0.5: resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} dependencies: split2: 4.2.0 + dev: false /picocolors@1.0.0: resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} @@ -11981,46 +12012,45 @@ packages: /postgres-array@2.0.0: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} + dev: false /postgres-array@3.0.2: resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} engines: {node: '>=12'} - dev: true /postgres-bytea@1.0.0: resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} engines: {node: '>=0.10.0'} + dev: false /postgres-bytea@3.0.0: resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} engines: {node: '>= 6'} dependencies: obuf: 1.1.2 - dev: true /postgres-date@1.0.7: resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} engines: {node: '>=0.10.0'} + dev: false /postgres-date@2.0.1: resolution: {integrity: sha512-YtMKdsDt5Ojv1wQRvUhnyDJNSr2dGIC96mQVKz7xufp07nfuFONzdaowrMHjlAzY6GDLd4f+LUHHAAM1h4MdUw==} engines: {node: '>=12'} - dev: true /postgres-interval@1.2.0: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} dependencies: xtend: 4.0.2 + dev: false /postgres-interval@3.0.0: resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} engines: {node: '>=12'} - dev: true /postgres-range@1.1.3: resolution: {integrity: sha512-VdlZoocy5lCP0c/t66xAfclglEapXPCIVhqqJRncYpvbCgImF0w67aPKfbqUMr72tO2k5q0TdTZwCLjPTI6C9g==} - dev: true /posthog-node@4.0.0: resolution: {integrity: sha512-jEZnNbgb/3FNk+gNwtTcyz3j+62zIN+UTPotONfacVXJnoI70KScSkKdIR+rvP9tA2kjBSoHQxGwJuizs27o9A==} @@ -12049,6 +12079,7 @@ packages: simple-get: 4.0.1 tar-fs: 2.1.1 tunnel-agent: 0.6.0 + dev: false /preferred-pm@3.1.2: resolution: {integrity: sha512-nk7dKrcW8hfCZ4H6klWcdRknBOXWzNQByJ0oJyX97BOupsYD+FzLS4hflgEu/uPUEHZCuRfMxzCBsuWd7OzT8Q==} @@ -12182,6 +12213,7 @@ packages: dependencies: end-of-stream: 1.4.4 once: 1.4.0 + dev: false /punycode@1.4.1: resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} @@ -12240,6 +12272,7 @@ packages: ini: 1.3.8 minimist: 1.2.8 strip-json-comments: 2.0.1 + dev: false /react-countup@6.5.0(react@18.2.0): resolution: {integrity: sha512-26JFHbUHsHxu8SetkJwWVIUEkaNnrj4P9msxNGC8tS4hGr1bngRzbwtJYOgXD2G/ItjaKJ3JfYKd85sw7qRVeA==} @@ -12960,6 +12993,7 @@ packages: /simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + dev: false /simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} @@ -12967,6 +13001,7 @@ packages: decompress-response: 6.0.0 once: 1.4.0 simple-concat: 1.0.1 + dev: false /simple-git-hooks@2.9.0: resolution: {integrity: sha512-waSQ5paUQtyGC0ZxlHmcMmD9I1rRXauikBwX31bX58l5vTOhCEcBC5Bi+ZDkPXTjDnZAF8TbCqKBY+9+sVPScw==} @@ -13134,6 +13169,7 @@ packages: /split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} + dev: false /sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} @@ -13357,6 +13393,7 @@ packages: /strip-json-comments@2.0.1: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} + dev: false /strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} @@ -13549,6 +13586,7 @@ packages: mkdirp-classic: 0.5.3 pump: 3.0.0 tar-stream: 2.2.0 + dev: false /tar-stream@1.6.2: resolution: {integrity: sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==} @@ -13572,6 +13610,7 @@ packages: fs-constants: 1.0.0 inherits: 2.0.4 readable-stream: 3.6.2 + dev: false /tar@6.2.0: resolution: {integrity: sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==} From 87f51371026b0fe67972b375cd7d7ab42fcabef1 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 24 Jun 2024 21:09:01 -0400 Subject: [PATCH 051/122] re-export drizzle objects --- examples/reference-erc20/src/api/index.ts | 12 ++++-- packages/core/src/build/plugin.ts | 1 + packages/core/src/index.ts | 5 ++- packages/core/src/server/service.ts | 51 ++++------------------- 4 files changed, 21 insertions(+), 48 deletions(-) diff --git a/examples/reference-erc20/src/api/index.ts b/examples/reference-erc20/src/api/index.ts index e12447519..bae89c0d9 100644 --- a/examples/reference-erc20/src/api/index.ts +++ b/examples/reference-erc20/src/api/index.ts @@ -1,6 +1,7 @@ -import { Account } from "ponder:db"; -import { ponder } from "ponder:virtual"; +import { Account, eq } from "ponder:db"; +import { ponder } from "@/generated"; import { graphql } from "@ponder/core"; +import { zeroAddress } from "viem"; // write file ponder.use("/graphql", graphql()); @@ -8,11 +9,14 @@ ponder.use("/graphql", graphql()); ponder.get("/router", async (c) => { const db = c.get("db"); - const account = await db.select().from(Account); + const account = await db + .select() + .from(Account) + .where(eq(Account.id, zeroAddress)); if (account.length === 0) { return c.text("Not Found!"); } else { - return c.text(`Balance: ${account[0]!.balance.toString()}`); + return c.text(`Balance: ${account[0]!.id.toString()}`); } }); diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 75abdf6fe..c9d50b7f0 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -41,6 +41,7 @@ let drizzleTables = Object.fromEntries( convertToDrizzleTable(tableName, table.table, "sqlite"), ]), ); +export * from "../../packages/core/src/drizzle/virtual"; module.exports = drizzleTables; `; } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 97d02bcac..e686f7dd6 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -21,4 +21,7 @@ export type DatabaseConfig = Prettify; export { graphql } from "@/graphql/index.js"; -export type { ConvertToDrizzleTable } from "@/drizzle/index.js"; +export { + type ConvertToDrizzleTable, + convertToDrizzleTable, +} from "@/drizzle/index.js"; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 6f47d0c14..8270c4447 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -1,17 +1,16 @@ import http from "node:http"; import type { Common } from "@/common/common.js"; +import { createDrizzleDb } from "@/drizzle/runtime.js"; import type { ReadonlyStore } from "@/indexing-store/store.js"; import type { Schema } from "@/schema/common.js"; -import { getTables } from "@/schema/utils.js"; -import type { DatabaseModel } from "@/types/model.js"; -import type { UserRecord } from "@/types/schema.js"; +import type { SqliteDatabase } from "@/utils/sqlite.js"; import { startClock } from "@/utils/timer.js"; import { serve } from "@hono/node-server"; import { Hono } from "hono"; import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; -import type { QueryResult, RawBuilder } from "kysely"; +import type { Pool } from "pg"; import { onError } from "./error.js"; type Server = { @@ -25,13 +24,15 @@ export async function createServer({ app: userApp, schema, readonlyStore, - query, + database, common, }: { app?: Hono; schema: Schema; readonlyStore: ReadonlyStore; - query: (query: RawBuilder) => Promise>; + database: + | { kind: "postgres"; pool: Pool } + | { kind: "sqlite"; database: SqliteDatabase }; common: Common; }): Promise { // Create hono app @@ -104,43 +105,7 @@ export async function createServer({ } }); - const db = Object.keys(getTables(schema)).reduce<{ - [tableName: string]: Pick< - DatabaseModel, - "findUnique" | "findMany" - >; - }>((acc, tableName) => { - acc[tableName] = { - findUnique: async ({ id }) => { - common.logger.trace({ - service: "store", - msg: `${tableName}.findUnique(id=${id})`, - }); - return readonlyStore.findUnique({ - tableName, - id, - }); - }, - findMany: async ({ where, orderBy, limit, before, after } = {}) => { - common.logger.trace({ - service: "store", - msg: `${tableName}.findMany`, - }); - return readonlyStore.findMany({ - tableName, - where, - orderBy, - limit, - before, - after, - }); - }, - }; - return acc; - }, {}); - - // @ts-ignore - db.query = query; + const db = createDrizzleDb(database); const contextMiddleware = createMiddleware(async (c, next) => { c.set("db", db); From a9a00e3d9c26ea34ed7e6eb9c41144b571b143cb Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 24 Jun 2024 21:23:51 -0400 Subject: [PATCH 052/122] wip --- examples/reference-erc20/ponder-env.d.ts | 17 ++++++++++++ packages/core/package.json | 10 +++++-- packages/core/src/build/plugin.ts | 2 +- packages/core/src/drizzle/table.ts | 2 ++ packages/core/src/drizzle/virtual.d.ts | 35 ++++++++++++++++++++++++ packages/core/tsup.config.ts | 2 +- 6 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 packages/core/src/drizzle/virtual.d.ts diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index f8e7347cf..667acccdd 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -3,6 +3,8 @@ // If this happens, please commit the changes. Do not manually edit this file. // See https://ponder.sh/docs/getting-started/installation#typescript for more information. +/// + declare module "@/generated" { import type { Virtual } from "@ponder/core"; @@ -25,3 +27,18 @@ declare module "@/generated" { Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; } + +declare module "ponder:db" { + import type { ConvertToDrizzleTable } from "@ponder/core"; + + type schema = typeof import("./ponder.schema.ts").default; + + const drizzleTables: { + [tableName in keyof schema]: ConvertToDrizzleTable< + tableName, + schema[tableName]["table"] + >; + }; + + export = drizzleTables; +} diff --git a/packages/core/package.json b/packages/core/package.json index 3073f515c..bc2c1a171 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -23,8 +23,14 @@ "types": "./dist/index.d.ts", "typings": "./dist/index.d.ts", "exports": { - "import": "./dist/index.js", - "types": "./dist/index.d.ts" + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + }, + "./virtual": { + "import": "./src/drizzle/virtual.js", + "types": "./src/drizzle/virtual.d.ts" + } }, "scripts": { "build": "tsup", diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index c9d50b7f0..bae6fad61 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -41,7 +41,7 @@ let drizzleTables = Object.fromEntries( convertToDrizzleTable(tableName, table.table, "sqlite"), ]), ); -export * from "../../packages/core/src/drizzle/virtual"; +export * from "@ponder/core/drizzle/virtual"; module.exports = drizzleTables; `; } diff --git a/packages/core/src/drizzle/table.ts b/packages/core/src/drizzle/table.ts index 8fd52074a..8b3a53e0b 100644 --- a/packages/core/src/drizzle/table.ts +++ b/packages/core/src/drizzle/table.ts @@ -46,9 +46,11 @@ export type ConvertToDrizzleTable< name: columnName & string; dataType: "string"; columnType: "custom"; + // @ts-ignore data: InferScalarType; driverParam: unknown; enumValues: undefined; + // @ts-ignore notNull: table[columnName][" optional"] extends true ? false : true; primaryKey: columnName extends "id" ? true : false; }>; diff --git a/packages/core/src/drizzle/virtual.d.ts b/packages/core/src/drizzle/virtual.d.ts new file mode 100644 index 000000000..73d1bcef5 --- /dev/null +++ b/packages/core/src/drizzle/virtual.d.ts @@ -0,0 +1,35 @@ +declare module "ponder:db" { + type RuntimeConfig = typeof import("./virtual.js"); + + export const sql: RuntimeConfig["sql"]; + + export const eq: RuntimeConfig["eq"]; + export const gt: RuntimeConfig["gt"]; + export const gte: RuntimeConfig["gte"]; + export const lt: RuntimeConfig["lt"]; + export const lte: RuntimeConfig["lte"]; + export const ne: RuntimeConfig["ne"]; + export const isNull: RuntimeConfig["isNull"]; + export const isNotNull: RuntimeConfig["isNotNull"]; + export const inArray: RuntimeConfig["inArray"]; + export const notInArray: RuntimeConfig["notInArray"]; + export const exists: RuntimeConfig["exists"]; + export const notExists: RuntimeConfig["notExists"]; + export const between: RuntimeConfig["between"]; + export const notBetween: RuntimeConfig["notBetween"]; + export const like: RuntimeConfig["like"]; + export const notIlike: RuntimeConfig["notIlike"]; + export const not: RuntimeConfig["not"]; + export const asc: RuntimeConfig["asc"]; + export const desc: RuntimeConfig["desc"]; + export const and: RuntimeConfig["and"]; + export const or: RuntimeConfig["or"]; + export const count: RuntimeConfig["count"]; + export const countDistinct: RuntimeConfig["countDistinct"]; + export const avg: RuntimeConfig["avg"]; + export const avgDistinct: RuntimeConfig["avgDistinct"]; + export const sum: RuntimeConfig["sum"]; + export const sumDistinct: RuntimeConfig["sumDistinct"]; + export const max: RuntimeConfig["max"]; + export const min: RuntimeConfig["min"]; +} diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index f261292a0..4cf94887a 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ name: "@ponder/core", - entry: ["src/index.ts", "src/bin/ponder.ts"], + entry: ["src/index.ts", "src/bin/ponder.ts", "src/drizzle/virtual.ts"], outDir: "dist", format: ["esm"], sourcemap: true, From 8685c9e4bf61c370c8efdaf133a524bad28a7cd1 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 25 Jun 2024 16:13:45 -0400 Subject: [PATCH 053/122] bigdog --- examples/reference-erc20/src/api/index.ts | 4 +- packages/core/package.json | 2 +- packages/core/src/build/plugin.ts | 2 +- packages/core/src/common/codegen.ts | 2 + .../core/src/database/postgres/service.ts | 2 +- packages/core/src/drizzle/bigint.ts | 51 +++++++++ packages/core/src/drizzle/hex.ts | 94 +++++++++++++++ packages/core/src/drizzle/runtime.test.ts | 87 +++++++++++--- packages/core/src/drizzle/runtime.ts | 108 ++++++++++++++++-- 9 files changed, 322 insertions(+), 30 deletions(-) create mode 100644 packages/core/src/drizzle/bigint.ts create mode 100644 packages/core/src/drizzle/hex.ts diff --git a/examples/reference-erc20/src/api/index.ts b/examples/reference-erc20/src/api/index.ts index bae89c0d9..c66f16530 100644 --- a/examples/reference-erc20/src/api/index.ts +++ b/examples/reference-erc20/src/api/index.ts @@ -10,13 +10,13 @@ ponder.get("/router", async (c) => { const db = c.get("db"); const account = await db - .select() + .select({ balance: Account.balance }) .from(Account) .where(eq(Account.id, zeroAddress)); if (account.length === 0) { return c.text("Not Found!"); } else { - return c.text(`Balance: ${account[0]!.id.toString()}`); + return c.text(`Balance: ${account[0]!.balance.toString()}`); } }); diff --git a/packages/core/package.json b/packages/core/package.json index bc2c1a171..59fb4360d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -28,7 +28,7 @@ "types": "./dist/index.d.ts" }, "./virtual": { - "import": "./src/drizzle/virtual.js", + "import": "./dist/drizzle/virtual.js", "types": "./src/drizzle/virtual.d.ts" } }, diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index bae6fad61..eb936f804 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -41,7 +41,7 @@ let drizzleTables = Object.fromEntries( convertToDrizzleTable(tableName, table.table, "sqlite"), ]), ); -export * from "@ponder/core/drizzle/virtual"; +export * from "@ponder/core/virtual"; module.exports = drizzleTables; `; } diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index 17205342a..bd3f1038c 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -8,6 +8,8 @@ export const ponderEnv = `// This file enables type checking and editor autocomp // If this happens, please commit the changes. Do not manually edit this file. // See https://ponder.sh/docs/getting-started/installation#typescript for more information. +/// + declare module "@/generated" { import type { Virtual } from "@ponder/core"; diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index 6f8c749fa..afc56394b 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -67,7 +67,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { private internalPool: Pool; private syncPool: Pool; private indexingPool: Pool; - private readonlyPool: Pool; + readonlyPool: Pool; constructor({ common, diff --git a/packages/core/src/drizzle/bigint.ts b/packages/core/src/drizzle/bigint.ts new file mode 100644 index 000000000..8ad7e8823 --- /dev/null +++ b/packages/core/src/drizzle/bigint.ts @@ -0,0 +1,51 @@ +import { decodeToBigInt, encodeAsText } from "@/utils/encoding.js"; +import { + type ColumnBaseConfig, + type ColumnBuilderBaseConfig, + type ColumnBuilderRuntimeConfig, + type MakeColumnConfig, + entityKind, +} from "drizzle-orm"; +import { + type AnySQLiteTable, + SQLiteColumn, + SQLiteColumnBuilder, +} from "drizzle-orm/sqlite-core"; + +export class SQLiteBigintBuilder< + T extends ColumnBuilderBaseConfig<"string", "SQLiteBigint">, +> extends SQLiteColumnBuilder { + static readonly [entityKind]: string = "SQliteHexBuilder"; + + constructor(name: T["name"]) { + super(name, "string", "SQLiteBigint"); + } + + /** @internal */ + build( + table: AnySQLiteTable<{ name: TTableName }>, + ): SQLiteBigint> { + return new SQLiteBigint>( + table, + this.config as ColumnBuilderRuntimeConfig, + ); + } +} + +export class SQLiteBigint< + T extends ColumnBaseConfig<"string", "SQLiteBigint">, +> extends SQLiteColumn { + static readonly [entityKind]: string = "SQLiteBigint"; + + getSQLType(): string { + return "varchar(79)"; + } + + override mapFromDriverValue(value: string): T["data"] { + return decodeToBigInt(value); + } + + override mapToDriverValue(value: T["data"]): string { + return encodeAsText(value as bigint); + } +} diff --git a/packages/core/src/drizzle/hex.ts b/packages/core/src/drizzle/hex.ts new file mode 100644 index 000000000..7fbbbc770 --- /dev/null +++ b/packages/core/src/drizzle/hex.ts @@ -0,0 +1,94 @@ +import { + type ColumnBaseConfig, + type ColumnBuilderBaseConfig, + type ColumnBuilderRuntimeConfig, + type MakeColumnConfig, + entityKind, +} from "drizzle-orm"; +import { + type AnyPgTable, + PgColumn, + PgColumnBuilder, +} from "drizzle-orm/pg-core"; +import { + type AnySQLiteTable, + SQLiteColumn, + SQLiteColumnBuilder, +} from "drizzle-orm/sqlite-core"; +import { bytesToHex, hexToBytes } from "viem"; + +export class PgHexBuilder< + T extends ColumnBuilderBaseConfig<"buffer", "PgHex">, +> extends PgColumnBuilder { + static readonly [entityKind]: string = "PgHexBuilder"; + + constructor(name: T["name"]) { + super(name, "buffer", "PgHex"); + } + + /** @internal */ + build( + table: AnyPgTable<{ name: TTableName }>, + ): PgHex> { + return new PgHex>( + table, + this.config as ColumnBuilderRuntimeConfig, + ); + } +} + +export class PgHex< + T extends ColumnBaseConfig<"buffer", "PgHex">, +> extends PgColumn { + static readonly [entityKind]: string = "PgHex"; + + getSQLType(): string { + return "bytea"; + } + + override mapFromDriverValue(value: Buffer): T["data"] { + return bytesToHex(value); + } + + override mapToDriverValue(value: T["data"]): Buffer { + return Buffer.from(hexToBytes(value as `0x${string}`)); + } +} + +export class SQLiteHexBuilder< + T extends ColumnBuilderBaseConfig<"buffer", "SQLiteHex">, +> extends SQLiteColumnBuilder { + static readonly [entityKind]: string = "SQliteHexBuilder"; + + constructor(name: T["name"]) { + super(name, "buffer", "SQLiteHex"); + } + + /** @internal */ + build( + table: AnySQLiteTable<{ name: TTableName }>, + ): SQLiteHex> { + return new SQLiteHex>( + table, + this.config as ColumnBuilderRuntimeConfig, + ); + } +} + +export class SQLiteHex< + T extends ColumnBaseConfig<"buffer", "SQLiteHex">, +> extends SQLiteColumn { + static readonly [entityKind]: string = "SQLiteHex"; + + getSQLType(): string { + return "blob"; + } + + override mapFromDriverValue(value: Buffer): T["data"] { + return bytesToHex(value); + } + + override mapToDriverValue(value: T["data"]): Buffer { + return Buffer.from(hexToBytes(value as `0x${string}`)); + } +} diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index ed1a1168d..bc8644bb8 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -3,16 +3,30 @@ import { setupDatabaseServices, setupIsolatedDatabase, } from "@/_test/setup.js"; +import type { DatabaseService } from "@/database/service.js"; import { SqliteDatabaseService } from "@/database/sqlite/service.js"; import type { HistoricalStore } from "@/indexing-store/store.js"; import { createSchema } from "@/schema/schema.js"; import { beforeEach, expect, test } from "vitest"; -import type { DrizzleDb } from "./db.js"; import { convertToDrizzleTable, createDrizzleDb } from "./runtime.js"; beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); +const createDb = (database: DatabaseService) => { + if (database instanceof SqliteDatabaseService) { + return createDrizzleDb({ + kind: "sqlite", + database: database.userDatabase, + }) as any; + } else { + return createDrizzleDb({ + kind: "postgres", + pool: database.readonlyPool, + }) as any; + } +}; + test("runtime select", async (context) => { const schema = createSchema((p) => ({ table: p.createTable({ @@ -22,31 +36,74 @@ test("runtime select", async (context) => { const { database, cleanup, indexingStore } = await setupDatabaseServices( context, - { - schema, - }, + { schema }, ); - let db: DrizzleDb; - - if (database instanceof SqliteDatabaseService) { - db = createDrizzleDb({ - kind: "sqlite", - database: database.userDatabase, - }) as any; - } + const db = createDb(database); await indexingStore.create({ tableName: "table", id: "kyle" }); - await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); - const rows = await db! + const rows = await db .select() .from(convertToDrizzleTable("table", schema.table.table, "sqlite")); expect(rows).toHaveLength(1); - expect(rows[0]).toMatchObject({ id: "kyle" }); await cleanup(); }); + +test("runtime hex", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.hex(), + }), + })); + + const { database, cleanup, indexingStore } = await setupDatabaseServices( + context, + { schema }, + ); + + const db = createDb(database); + + await indexingStore.create({ tableName: "table", id: "0x1" }); + await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); + + const rows = await db + .select() + .from(convertToDrizzleTable("table", schema.table.table, "postgres")); + + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: "0x01" }); + + await cleanup(); +}); + +test("runtime bigint", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.bigint(), + }), + })); + + const { database, cleanup, indexingStore } = await setupDatabaseServices( + context, + { schema }, + ); + + const db = createDb(database); + + await indexingStore.create({ tableName: "table", id: 1n }); + await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); + + const rows = await db + .select() + .from(convertToDrizzleTable("table", schema.table.table, "sqlite")); + + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: 1n }); + + await cleanup(); +}); diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index 7c8728518..81fa840f9 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -10,12 +10,21 @@ import type { Table } from "drizzle-orm"; import { drizzle as drizzleSQLite } from "drizzle-orm/better-sqlite3"; import { drizzle as drizzlePg } from "drizzle-orm/node-postgres"; import { pgTable } from "drizzle-orm/pg-core"; +import { + doublePrecision as PgDoublePrecision, + integer as PgInteger, + numeric as PgNumeric, + text as PgText, +} from "drizzle-orm/pg-core"; import { integer as SQLiteInteger, + real as SQLiteReal, text as SQLiteText, sqliteTable, } from "drizzle-orm/sqlite-core"; import type { Pool } from "pg"; +import { SQLiteBigintBuilder } from "./bigint.js"; +import { PgHexBuilder, SQLiteHexBuilder } from "./hex.js"; export const createDrizzleDb = ( database: @@ -40,6 +49,8 @@ export const createDrizzleDb = ( } }; +// TODO: handle schemas + export const convertToDrizzleTable = ( tableName: string, table: PonderTable, @@ -49,21 +60,38 @@ export const convertToDrizzleTable = ( (acc, [columnName, column]) => { if (isMaterialColumn(column)) { if (isScalarColumn(column) || isReferenceColumn(column)) { - let drizzleColumn = - column[" scalar"] === "string" - ? SQLiteText(columnName) - : column[" scalar"] === "int" - ? SQLiteInteger(columnName) - : undefined; + switch (column[" scalar"]) { + case "string": + acc[columnName] = convertStringColumn(columnName, kind); + break; + + case "int": + acc[columnName] = convertIntColumn(columnName, kind); + break; + + case "boolean": + acc[columnName] = convertBooleanColumn(columnName, kind); + break; + + case "float": + acc[columnName] = convertFloatColumn(columnName, kind); + break; + + case "hex": + acc[columnName] = convertHexColumn(columnName, kind); + break; + + case "bigint": + acc[columnName] = convertBigintColumn(columnName, kind); + break; + } // apply column constraints if (columnName === "id") { - drizzleColumn = drizzleColumn!.primaryKey(); + acc[columnName] = acc[columnName]!.primaryKey(); } else if (isOptionalColumn(column) === false) { - drizzleColumn = drizzleColumn!.notNull(); + acc[columnName] = acc[columnName]!.notNull(); } - - acc[columnName] = drizzleColumn; } } return acc; @@ -77,3 +105,63 @@ export const convertToDrizzleTable = ( return sqliteTable(tableName, columns); } }; + +const convertStringColumn = ( + columnName: string, + kind: "sqlite" | "postgres", +) => { + return kind === "sqlite" ? SQLiteText(columnName) : PgText(columnName); +}; + +const convertIntColumn = (columnName: string, kind: "sqlite" | "postgres") => { + return kind === "sqlite" ? SQLiteInteger(columnName) : PgInteger(columnName); +}; + +const convertFloatColumn = ( + columnName: string, + kind: "sqlite" | "postgres", +) => { + return kind === "sqlite" + ? SQLiteReal(columnName) + : PgDoublePrecision(columnName); +}; + +const convertBooleanColumn = ( + columnName: string, + kind: "sqlite" | "postgres", +) => { + return kind === "sqlite" ? SQLiteInteger(columnName) : PgInteger(columnName); +}; + +const convertHexColumn = (columnName: string, kind: "sqlite" | "postgres") => { + return kind === "sqlite" + ? new SQLiteHexBuilder(columnName) + : new PgHexBuilder(columnName); +}; + +const convertBigintColumn = ( + columnName: string, + kind: "sqlite" | "postgres", +) => { + return kind === "sqlite" + ? new SQLiteBigintBuilder(columnName) + : PgNumeric(columnName, { precision: 78 }); +}; + +// const convertListColumn = ( +// tableName: string, +// columnName: string, +// column: ScalarColumn<"bigint">, +// ) => {}; + +// const convertJsonColumn = ( +// tableName: string, +// columnName: string, +// column: ScalarColumn<"bigint">, +// ) => {}; + +// const convertEnumColumn = ( +// tableName: string, +// columnName: string, +// column: ScalarColumn<"bigint">, +// ) => {}; From f6e671300f9757f54f07e2ffcb2ae900b451ab6c Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 25 Jun 2024 17:34:55 -0400 Subject: [PATCH 054/122] more --- packages/core/package.json | 2 +- packages/core/src/build/plugin.ts | 4 +++- packages/core/src/drizzle/db.ts | 2 +- packages/core/src/drizzle/table.ts | 19 +++++++++++++------ 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/core/package.json b/packages/core/package.json index 59fb4360d..edf742ef2 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -29,7 +29,7 @@ }, "./virtual": { "import": "./dist/drizzle/virtual.js", - "types": "./src/drizzle/virtual.d.ts" + "types": "./dist/drizzle/virtual.d.ts" } }, "scripts": { diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index eb936f804..68ac18b9a 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -29,6 +29,8 @@ export function replaceStateless(code: string, regex: RegExp, shim: string) { } export const vitePluginPonder = (): Plugin => { + const databasekind = "sqlite"; + return { name: "ponder", load: (id) => { @@ -38,7 +40,7 @@ import { convertToDrizzleTable } from "@ponder/core"; let drizzleTables = Object.fromEntries( Object.entries(schema).map(([tableName, table]) => [ tableName, - convertToDrizzleTable(tableName, table.table, "sqlite"), + convertToDrizzleTable(tableName, table.table, "${databasekind}"), ]), ); export * from "@ponder/core/virtual"; diff --git a/packages/core/src/drizzle/db.ts b/packages/core/src/drizzle/db.ts index 99edbe375..9f6f108db 100644 --- a/packages/core/src/drizzle/db.ts +++ b/packages/core/src/drizzle/db.ts @@ -11,5 +11,5 @@ export type DrizzleDb = { ): SelectBuilder | undefined, "async", void>; execute: >( query: SQLWrapper, - ) => record[]; + ) => Promise; }; diff --git a/packages/core/src/drizzle/table.ts b/packages/core/src/drizzle/table.ts index 8b3a53e0b..c9f2d2845 100644 --- a/packages/core/src/drizzle/table.ts +++ b/packages/core/src/drizzle/table.ts @@ -1,6 +1,8 @@ import type { ExtractNonVirtualColumnNames, Table as PonderTable, + ReferenceColumn, + ScalarColumn, } from "@/schema/common.js"; import type { InferScalarType } from "@/schema/infer.js"; import type { @@ -31,6 +33,8 @@ export type ViewWithSelection< > = View & TSelection; /** + * Performs type transformation between Ponder and Drizzle column representation. + * * @returns TableWithColumns */ export type ConvertToDrizzleTable< @@ -44,14 +48,17 @@ export type ConvertToDrizzleTable< { [columnName in ExtractNonVirtualColumnNames
]: ColumnBuilderBase<{ name: columnName & string; - dataType: "string"; - columnType: "custom"; - // @ts-ignore - data: InferScalarType; + dataType: "custom"; + columnType: "ponder"; + data: InferScalarType< + (table[columnName] & (ScalarColumn | ReferenceColumn))[" scalar"] + >; driverParam: unknown; enumValues: undefined; - // @ts-ignore - notNull: table[columnName][" optional"] extends true ? false : true; + notNull: (table[columnName] & + (ScalarColumn | ReferenceColumn))[" optional"] extends true + ? false + : true; primaryKey: columnName extends "id" ? true : false; }>; }, From c376e9050321683958d395802b648fdc6447606c Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 26 Jun 2024 14:17:52 -0400 Subject: [PATCH 055/122] more column types --- examples/reference-erc20/ponder-env.d.ts | 3 +- packages/core/src/common/codegen.ts | 3 +- packages/core/src/drizzle/bigint.ts | 2 +- packages/core/src/drizzle/json.ts | 50 +++++++++ packages/core/src/drizzle/runtime.test.ts | 75 ++++++++++++- packages/core/src/drizzle/runtime.ts | 38 ++++--- packages/core/src/drizzle/select.ts | 131 ++++++++++++++++++++++ packages/core/src/drizzle/table.ts | 8 +- 8 files changed, 282 insertions(+), 28 deletions(-) create mode 100644 packages/core/src/drizzle/json.ts diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index 667acccdd..1c6c13d45 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -36,7 +36,8 @@ declare module "ponder:db" { const drizzleTables: { [tableName in keyof schema]: ConvertToDrizzleTable< tableName, - schema[tableName]["table"] + schema[tableName]["table"], + schema >; }; diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index bd3f1038c..a5fecdf47 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -41,7 +41,8 @@ declare module "ponder:db" { const drizzleTables: { [tableName in keyof schema]: ConvertToDrizzleTable< tableName, - schema[tableName]["table"] + schema[tableName]["table"], + schema, >; }; diff --git a/packages/core/src/drizzle/bigint.ts b/packages/core/src/drizzle/bigint.ts index 8ad7e8823..4911ce841 100644 --- a/packages/core/src/drizzle/bigint.ts +++ b/packages/core/src/drizzle/bigint.ts @@ -15,7 +15,7 @@ import { export class SQLiteBigintBuilder< T extends ColumnBuilderBaseConfig<"string", "SQLiteBigint">, > extends SQLiteColumnBuilder { - static readonly [entityKind]: string = "SQliteHexBuilder"; + static readonly [entityKind]: string = "SQliteBigintBuilder"; constructor(name: T["name"]) { super(name, "string", "SQLiteBigint"); diff --git a/packages/core/src/drizzle/json.ts b/packages/core/src/drizzle/json.ts new file mode 100644 index 000000000..7a90cd1bd --- /dev/null +++ b/packages/core/src/drizzle/json.ts @@ -0,0 +1,50 @@ +import { + type ColumnBaseConfig, + type ColumnBuilderBaseConfig, + type ColumnBuilderRuntimeConfig, + type MakeColumnConfig, + entityKind, +} from "drizzle-orm"; +import { + type AnySQLiteTable, + SQLiteColumn, + SQLiteColumnBuilder, +} from "drizzle-orm/sqlite-core"; + +export class SQLiteJsonBuilder< + T extends ColumnBuilderBaseConfig<"json", "SQLiteJson">, +> extends SQLiteColumnBuilder { + static readonly [entityKind]: string = "SQliteJsonBuilder"; + + constructor(name: T["name"]) { + super(name, "json", "SQLiteJson"); + } + + /** @internal */ + build( + table: AnySQLiteTable<{ name: TTableName }>, + ): SQLiteJson> { + return new SQLiteJson>( + table, + this.config as ColumnBuilderRuntimeConfig, + ); + } +} + +export class SQLiteJson< + T extends ColumnBaseConfig<"json", "SQLiteJson">, +> extends SQLiteColumn { + static readonly [entityKind]: string = "SQLiteJson"; + + getSQLType(): string { + return "jsonb"; + } + + override mapFromDriverValue(value: string): T["data"] { + return JSON.parse(value); + } + + override mapToDriverValue(value: T["data"]): string { + return JSON.stringify(value); + } +} diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index bc8644bb8..e127b76df 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -16,7 +16,7 @@ beforeEach(setupIsolatedDatabase); const createDb = (database: DatabaseService) => { if (database instanceof SqliteDatabaseService) { return createDrizzleDb({ - kind: "sqlite", + kind: database.kind, database: database.userDatabase, }) as any; } else { @@ -46,7 +46,7 @@ test("runtime select", async (context) => { const rows = await db .select() - .from(convertToDrizzleTable("table", schema.table.table, "sqlite")); + .from(convertToDrizzleTable("table", schema.table.table, database.kind)); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: "kyle" }); @@ -100,10 +100,79 @@ test("runtime bigint", async (context) => { const rows = await db .select() - .from(convertToDrizzleTable("table", schema.table.table, "sqlite")); + .from(convertToDrizzleTable("table", schema.table.table, database.kind)); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: 1n }); await cleanup(); }); + +test("runtime json", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.string(), + json: p.json(), + }), + })); + + const { database, cleanup, indexingStore } = await setupDatabaseServices( + context, + { schema }, + ); + + const db = createDb(database); + + await indexingStore.create({ + tableName: "table", + id: "1", + data: { + json: { + prop: 52, + }, + }, + }); + await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); + + const rows = await db + .select() + .from(convertToDrizzleTable("table", schema.table.table, database.kind)); + + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: "1", json: { prop: 52 } }); + + await cleanup(); +}); + +test("runtime enum", async (context) => { + const schema = createSchema((p) => ({ + en: p.createEnum(["hi", "low"]), + table: p.createTable({ + id: p.string(), + en: p.enum("en"), + }), + })); + + const { database, cleanup, indexingStore } = await setupDatabaseServices( + context, + { schema }, + ); + + const db = createDb(database); + + await indexingStore.create({ + tableName: "table", + id: "1", + data: { en: "hi" }, + }); + await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); + + const rows = await db + .select() + .from(convertToDrizzleTable("table", schema.table.table, database.kind)); + + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: "1", en: "hi" }); + + await cleanup(); +}); diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index 81fa840f9..086dac0f5 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -1,5 +1,7 @@ import type { Table as PonderTable } from "@/schema/common.js"; import { + isEnumColumn, + isJSONColumn, isMaterialColumn, isOptionalColumn, isReferenceColumn, @@ -13,6 +15,7 @@ import { pgTable } from "drizzle-orm/pg-core"; import { doublePrecision as PgDoublePrecision, integer as PgInteger, + jsonb as PgJsonb, numeric as PgNumeric, text as PgText, } from "drizzle-orm/pg-core"; @@ -25,6 +28,7 @@ import { import type { Pool } from "pg"; import { SQLiteBigintBuilder } from "./bigint.js"; import { PgHexBuilder, SQLiteHexBuilder } from "./hex.js"; +import { SQLiteJsonBuilder } from "./json.js"; export const createDrizzleDb = ( database: @@ -59,7 +63,11 @@ export const convertToDrizzleTable = ( const columns = Object.entries(table).reduce<{ [columnName: string]: any }>( (acc, [columnName, column]) => { if (isMaterialColumn(column)) { - if (isScalarColumn(column) || isReferenceColumn(column)) { + if (isJSONColumn(column)) { + acc[columnName] = convertJsonColumn(columnName, kind); + } else if (isEnumColumn(column)) { + acc[columnName] = convertEnumColumn(columnName, kind); + } else if (isScalarColumn(column) || isReferenceColumn(column)) { switch (column[" scalar"]) { case "string": acc[columnName] = convertStringColumn(columnName, kind); @@ -148,20 +156,14 @@ const convertBigintColumn = ( : PgNumeric(columnName, { precision: 78 }); }; -// const convertListColumn = ( -// tableName: string, -// columnName: string, -// column: ScalarColumn<"bigint">, -// ) => {}; - -// const convertJsonColumn = ( -// tableName: string, -// columnName: string, -// column: ScalarColumn<"bigint">, -// ) => {}; - -// const convertEnumColumn = ( -// tableName: string, -// columnName: string, -// column: ScalarColumn<"bigint">, -// ) => {}; +// TODO(kyle) list + +const convertJsonColumn = (columnName: string, kind: "sqlite" | "postgres") => { + return kind === "sqlite" + ? new SQLiteJsonBuilder(columnName) + : PgJsonb(columnName); +}; + +const convertEnumColumn = (columnName: string, kind: "sqlite" | "postgres") => { + return kind === "sqlite" ? SQLiteText(columnName) : PgText(columnName); +}; diff --git a/packages/core/src/drizzle/select.ts b/packages/core/src/drizzle/select.ts index b1e7776dc..6036e217d 100644 --- a/packages/core/src/drizzle/select.ts +++ b/packages/core/src/drizzle/select.ts @@ -12,6 +12,8 @@ import type { } from "drizzle-orm"; import type { TypedQueryBuilder } from "drizzle-orm/query-builders/query-builder"; import type { + AppendToNullabilityMap, + AppendToResult, BuildSubquerySelection, GetSelectTableName, GetSelectTableSelection, @@ -53,6 +55,63 @@ export type SelectJoinConfig = { joinType: JoinType; }; +export type Join< + T extends AnySelectQueryBuilder, + TDynamic extends boolean, + TJoinType extends JoinType, + TJoinedTable extends Table | Subquery | View | SQL, + TJoinedName extends + GetSelectTableName = GetSelectTableName, +> = T extends any + ? SelectWithout< + SelectKind< + T["_"]["hkt"], + T["_"]["tableName"], + T["_"]["resultType"], + T["_"]["runResult"], + AppendToResult< + T["_"]["tableName"], + T["_"]["selection"], + TJoinedName, + TJoinedTable extends Table + ? TJoinedTable["_"]["columns"] + : TJoinedTable extends Subquery | View + ? Assume< + TJoinedTable["_"]["selectedFields"], + SelectedFields + > + : never, + T["_"]["selectMode"] + >, + T["_"]["selectMode"] extends "partial" + ? T["_"]["selectMode"] + : "multiple", + AppendToNullabilityMap< + T["_"]["nullabilityMap"], + TJoinedName, + TJoinType + >, + T["_"]["dynamic"], + T["_"]["excludedMethods"] + >, + TDynamic, + T["_"]["excludedMethods"] + > + : never; + +export type JoinFn< + T extends AnySelectQueryBuilder, + TDynamic extends boolean, + TJoinType extends JoinType, +> = < + TJoinedTable extends Table | Subquery | View | SQL, + TJoinedName extends + GetSelectTableName = GetSelectTableName, +>( + table: TJoinedTable, + on: ((aliases: T["_"]["selection"]) => SQL | undefined) | SQL | undefined, +) => Join; + /** * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/query-builders/select.types.ts#L75 */ @@ -234,6 +293,78 @@ export type SelectQueryBuilderBase< readonly selectedFields: TSelectedFields; }; + leftJoin: JoinFn< + SelectQueryBuilderBase< + THKT, + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode, + TNullabilityMap, + TDynamic, + TExcludedMethods, + TResult, + TSelectedFields + >, + TDynamic, + "left" + >; + + rightJoin: JoinFn< + SelectQueryBuilderBase< + THKT, + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode, + TNullabilityMap, + TDynamic, + TExcludedMethods, + TResult, + TSelectedFields + >, + TDynamic, + "right" + >; + + innerJoin: JoinFn< + SelectQueryBuilderBase< + THKT, + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode, + TNullabilityMap, + TDynamic, + TExcludedMethods, + TResult, + TSelectedFields + >, + TDynamic, + "inner" + >; + + fullJoin: JoinFn< + SelectQueryBuilderBase< + THKT, + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode, + TNullabilityMap, + TDynamic, + TExcludedMethods, + TResult, + TSelectedFields + >, + TDynamic, + "full" + >; + where: ( where: ((aliases: TSelection) => SQL | undefined) | SQL | undefined, ) => SelectWithout< diff --git a/packages/core/src/drizzle/table.ts b/packages/core/src/drizzle/table.ts index c9f2d2845..4741b0ab1 100644 --- a/packages/core/src/drizzle/table.ts +++ b/packages/core/src/drizzle/table.ts @@ -1,10 +1,11 @@ import type { ExtractNonVirtualColumnNames, + Schema as PonderSchema, Table as PonderTable, ReferenceColumn, ScalarColumn, } from "@/schema/common.js"; -import type { InferScalarType } from "@/schema/infer.js"; +import type { InferColumnType } from "@/schema/infer.js"; import type { BuildColumns, ColumnBuilderBase, @@ -40,6 +41,7 @@ export type ViewWithSelection< export type ConvertToDrizzleTable< tableName extends string, table extends PonderTable, + schema extends PonderSchema, > = TableWithColumns<{ name: tableName; schema: undefined; @@ -50,9 +52,7 @@ export type ConvertToDrizzleTable< name: columnName & string; dataType: "custom"; columnType: "ponder"; - data: InferScalarType< - (table[columnName] & (ScalarColumn | ReferenceColumn))[" scalar"] - >; + data: InferColumnType; driverParam: unknown; enumValues: undefined; notNull: (table[columnName] & From f0f818dafe927019c7091264d0a56225dc5b171a Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 26 Jun 2024 17:34:33 -0400 Subject: [PATCH 056/122] cleanup + add all types --- examples/reference-erc20/src/api/index.ts | 11 +- packages/core/src/common/codegen.ts | 2 +- packages/core/src/drizzle/bigint.ts | 34 +- packages/core/src/drizzle/hex.ts | 64 +- packages/core/src/drizzle/json.ts | 34 +- packages/core/src/drizzle/runtime.ts | 1 - packages/core/src/drizzle/select.ts | 733 +++++++++++++++------- packages/core/src/drizzle/table.ts | 28 +- 8 files changed, 545 insertions(+), 362 deletions(-) diff --git a/examples/reference-erc20/src/api/index.ts b/examples/reference-erc20/src/api/index.ts index c66f16530..e07257726 100644 --- a/examples/reference-erc20/src/api/index.ts +++ b/examples/reference-erc20/src/api/index.ts @@ -1,22 +1,23 @@ -import { Account, eq } from "ponder:db"; +import { Account, desc } from "ponder:db"; import { ponder } from "@/generated"; import { graphql } from "@ponder/core"; -import { zeroAddress } from "viem"; +import { formatEther } from "viem"; // write file ponder.use("/graphql", graphql()); -ponder.get("/router", async (c) => { +ponder.get("/big", async (c) => { const db = c.get("db"); const account = await db .select({ balance: Account.balance }) .from(Account) - .where(eq(Account.id, zeroAddress)); + .orderBy(desc(Account.balance)) + .limit(1); if (account.length === 0) { return c.text("Not Found!"); } else { - return c.text(`Balance: ${account[0]!.balance.toString()}`); + return c.text(`Balance: ${formatEther(account[0]!.balance)}`); } }); diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index a5fecdf47..e2f9a2fc7 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -42,7 +42,7 @@ declare module "ponder:db" { [tableName in keyof schema]: ConvertToDrizzleTable< tableName, schema[tableName]["table"], - schema, + schema >; }; diff --git a/packages/core/src/drizzle/bigint.ts b/packages/core/src/drizzle/bigint.ts index 4911ce841..0e499df67 100644 --- a/packages/core/src/drizzle/bigint.ts +++ b/packages/core/src/drizzle/bigint.ts @@ -1,51 +1,35 @@ import { decodeToBigInt, encodeAsText } from "@/utils/encoding.js"; -import { - type ColumnBaseConfig, - type ColumnBuilderBaseConfig, - type ColumnBuilderRuntimeConfig, - type MakeColumnConfig, - entityKind, -} from "drizzle-orm"; +import { entityKind } from "drizzle-orm"; import { type AnySQLiteTable, SQLiteColumn, SQLiteColumnBuilder, } from "drizzle-orm/sqlite-core"; -export class SQLiteBigintBuilder< - T extends ColumnBuilderBaseConfig<"string", "SQLiteBigint">, -> extends SQLiteColumnBuilder { +export class SQLiteBigintBuilder extends SQLiteColumnBuilder { static readonly [entityKind]: string = "SQliteBigintBuilder"; - constructor(name: T["name"]) { - super(name, "string", "SQLiteBigint"); + constructor(columnName: string) { + super(columnName, "string", "SQLiteBigint"); } - /** @internal */ - build( - table: AnySQLiteTable<{ name: TTableName }>, - ): SQLiteBigint> { - return new SQLiteBigint>( - table, - this.config as ColumnBuilderRuntimeConfig, - ); + build(table: AnySQLiteTable) { + return new SQLiteBigint(table, this.config); } } -export class SQLiteBigint< - T extends ColumnBaseConfig<"string", "SQLiteBigint">, -> extends SQLiteColumn { +export class SQLiteBigint extends SQLiteColumn { static readonly [entityKind]: string = "SQLiteBigint"; getSQLType(): string { return "varchar(79)"; } - override mapFromDriverValue(value: string): T["data"] { + override mapFromDriverValue(value: string) { return decodeToBigInt(value); } - override mapToDriverValue(value: T["data"]): string { + override mapToDriverValue(value: bigint): string { return encodeAsText(value as bigint); } } diff --git a/packages/core/src/drizzle/hex.ts b/packages/core/src/drizzle/hex.ts index 7fbbbc770..40708fd7a 100644 --- a/packages/core/src/drizzle/hex.ts +++ b/packages/core/src/drizzle/hex.ts @@ -1,10 +1,4 @@ -import { - type ColumnBaseConfig, - type ColumnBuilderBaseConfig, - type ColumnBuilderRuntimeConfig, - type MakeColumnConfig, - entityKind, -} from "drizzle-orm"; +import { entityKind } from "drizzle-orm"; import { type AnyPgTable, PgColumn, @@ -17,78 +11,58 @@ import { } from "drizzle-orm/sqlite-core"; import { bytesToHex, hexToBytes } from "viem"; -export class PgHexBuilder< - T extends ColumnBuilderBaseConfig<"buffer", "PgHex">, -> extends PgColumnBuilder { +export class PgHexBuilder extends PgColumnBuilder { static readonly [entityKind]: string = "PgHexBuilder"; - constructor(name: T["name"]) { - super(name, "buffer", "PgHex"); + constructor(columnName: string) { + super(columnName, "buffer", "PgHex"); } - /** @internal */ - build( - table: AnyPgTable<{ name: TTableName }>, - ): PgHex> { - return new PgHex>( - table, - this.config as ColumnBuilderRuntimeConfig, - ); + build(table: AnyPgTable) { + return new PgHex(table, this.config); } } -export class PgHex< - T extends ColumnBaseConfig<"buffer", "PgHex">, -> extends PgColumn { +export class PgHex extends PgColumn { static readonly [entityKind]: string = "PgHex"; getSQLType(): string { return "bytea"; } - override mapFromDriverValue(value: Buffer): T["data"] { + override mapFromDriverValue(value: Buffer) { return bytesToHex(value); } - override mapToDriverValue(value: T["data"]): Buffer { - return Buffer.from(hexToBytes(value as `0x${string}`)); + override mapToDriverValue(value: `0x${string}`): Buffer { + return Buffer.from(hexToBytes(value)); } } -export class SQLiteHexBuilder< - T extends ColumnBuilderBaseConfig<"buffer", "SQLiteHex">, -> extends SQLiteColumnBuilder { +export class SQLiteHexBuilder extends SQLiteColumnBuilder { static readonly [entityKind]: string = "SQliteHexBuilder"; - constructor(name: T["name"]) { - super(name, "buffer", "SQLiteHex"); + constructor(columnName: string) { + super(columnName, "buffer", "SQLiteHex"); } - /** @internal */ - build( - table: AnySQLiteTable<{ name: TTableName }>, - ): SQLiteHex> { - return new SQLiteHex>( - table, - this.config as ColumnBuilderRuntimeConfig, - ); + build(table: AnySQLiteTable) { + return new SQLiteHex(table, this.config); } } -export class SQLiteHex< - T extends ColumnBaseConfig<"buffer", "SQLiteHex">, -> extends SQLiteColumn { +export class SQLiteHex extends SQLiteColumn { static readonly [entityKind]: string = "SQLiteHex"; getSQLType(): string { return "blob"; } - override mapFromDriverValue(value: Buffer): T["data"] { + override mapFromDriverValue(value: Buffer) { return bytesToHex(value); } - override mapToDriverValue(value: T["data"]): Buffer { - return Buffer.from(hexToBytes(value as `0x${string}`)); + override mapToDriverValue(value: `0x${string}`): Buffer { + return Buffer.from(hexToBytes(value)); } } diff --git a/packages/core/src/drizzle/json.ts b/packages/core/src/drizzle/json.ts index 7a90cd1bd..1503ce56f 100644 --- a/packages/core/src/drizzle/json.ts +++ b/packages/core/src/drizzle/json.ts @@ -1,50 +1,34 @@ -import { - type ColumnBaseConfig, - type ColumnBuilderBaseConfig, - type ColumnBuilderRuntimeConfig, - type MakeColumnConfig, - entityKind, -} from "drizzle-orm"; +import { entityKind } from "drizzle-orm"; import { type AnySQLiteTable, SQLiteColumn, SQLiteColumnBuilder, } from "drizzle-orm/sqlite-core"; -export class SQLiteJsonBuilder< - T extends ColumnBuilderBaseConfig<"json", "SQLiteJson">, -> extends SQLiteColumnBuilder { +export class SQLiteJsonBuilder extends SQLiteColumnBuilder { static readonly [entityKind]: string = "SQliteJsonBuilder"; - constructor(name: T["name"]) { - super(name, "json", "SQLiteJson"); + constructor(columnName: string) { + super(columnName, "json", "SQLiteJson"); } - /** @internal */ - build( - table: AnySQLiteTable<{ name: TTableName }>, - ): SQLiteJson> { - return new SQLiteJson>( - table, - this.config as ColumnBuilderRuntimeConfig, - ); + build(table: AnySQLiteTable) { + return new SQLiteJson(table, this.config); } } -export class SQLiteJson< - T extends ColumnBaseConfig<"json", "SQLiteJson">, -> extends SQLiteColumn { +export class SQLiteJson extends SQLiteColumn { static readonly [entityKind]: string = "SQLiteJson"; getSQLType(): string { return "jsonb"; } - override mapFromDriverValue(value: string): T["data"] { + override mapFromDriverValue(value: string) { return JSON.parse(value); } - override mapToDriverValue(value: T["data"]): string { + override mapToDriverValue(value: object): string { return JSON.stringify(value); } } diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index 086dac0f5..2356a6f38 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -37,7 +37,6 @@ export const createDrizzleDb = ( ) => { if (database.kind === "postgres") { const drizzle = drizzlePg(database.pool); - drizzle.execute; return { // @ts-ignore select: (...args: any[]) => drizzle.select(...args), diff --git a/packages/core/src/drizzle/select.ts b/packages/core/src/drizzle/select.ts index 6036e217d..76240ddb0 100644 --- a/packages/core/src/drizzle/select.ts +++ b/packages/core/src/drizzle/select.ts @@ -3,15 +3,20 @@ import type { Column, MakeColumnConfig, QueryPromise, - SelectedFieldsOrdered as SelectFieldsOrderedBase, SelectedFields, + SelectedFieldsOrdered, Subquery, Table, + TableConfig, UpdateTableConfig, + ValidateShape, + ValueOrArray, + // ValueOrArray, . entityKind, } from "drizzle-orm"; -import type { TypedQueryBuilder } from "drizzle-orm/query-builders/query-builder"; +import { TypedQueryBuilder } from "drizzle-orm/query-builders/query-builder"; import type { + AddAliasToSelection, AppendToNullabilityMap, AppendToResult, BuildSubquerySelection, @@ -23,9 +28,18 @@ import type { SelectResult, SetOperator, } from "drizzle-orm/query-builders/select.types"; -import type { ColumnsSelection, Placeholder, SQL, View } from "drizzle-orm/sql"; -import type { TableWithColumns, ViewWithSelection } from "./table.js"; +import type { + ColumnsSelection, + Placeholder, + Query, + SQL, + View, +} from "drizzle-orm/sql"; +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.ts#L54 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.ts#L50 + */ export type SelectBuilder< TSelection extends SelectedFields | undefined, TResultType extends "sync" | "async", @@ -44,6 +58,171 @@ export type SelectBuilder< >; }; +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.ts#L126 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.ts#L130 + */ +export abstract class SelectQueryBuilderBase< + THKT extends SelectHKTBase, + TTableName extends string | undefined, + TResultType extends "sync" | "async", + TRunResult, + TSelection extends ColumnsSelection, + TSelectMode extends SelectMode, + TNullabilityMap extends Record< + string, + JoinNullability + > = TTableName extends string ? Record : {}, + TDynamic extends boolean = false, + TExcludedMethods extends string = never, + TResult extends any[] = SelectResult< + TSelection, + TSelectMode, + TNullabilityMap + >[], + TSelectedFields extends ColumnsSelection = BuildSubquerySelection< + TSelection, + TNullabilityMap + >, +> extends TypedQueryBuilder { + declare [entityKind]: string; + declare _: { + readonly hkt: THKT; + readonly tableName: TTableName; + readonly resultType: TResultType; + readonly runResult: TRunResult; + readonly selection: TSelection; + readonly selectMode: TSelectMode; + readonly nullabilityMap: TNullabilityMap; + readonly dynamic: TDynamic; + readonly excludedMethods: TExcludedMethods; + readonly result: TResult; + readonly selectedFields: TSelectedFields; + }; + + declare leftJoin: JoinFn; + declare rightJoin: JoinFn; + declare innerJoin: JoinFn; + declare fullJoin: JoinFn; + + private declare setOperator: >( + rightSelection: + | (( + setOperators: GetSetOperators, + ) => SetOperatorRightSelect) + | SetOperatorRightSelect, + ) => SelectWithout; + + declare union: typeof this.setOperator; + declare unionAll: typeof this.setOperator; + declare intersect: typeof this.setOperator; + declare intersectAll: typeof this.setOperator; + declare except: typeof this.setOperator; + declare exceptAll: typeof this.setOperator; + + declare where: ( + where: ((aliases: TSelection) => SQL | undefined) | SQL | undefined, + ) => SelectWithout; + + declare having: ( + having: + | ((aliases: this["_"]["selection"]) => SQL | undefined) + | SQL + | undefined, + ) => SelectWithout; + + declare groupBy: + | (( + builder: ( + aliases: this["_"]["selection"], + ) => ValueOrArray, + ) => SelectWithout) + | (( + ...columns: (Column | SQL)[] + ) => SelectWithout) + | (( + ...columns: + | [ + ( + aliases: this["_"]["selection"], + ) => ValueOrArray, + ] + | (Column | SQL | SQL.Aliased)[] + ) => SelectWithout); + + declare orderBy: + | (( + builder: ( + aliases: this["_"]["selection"], + ) => ValueOrArray, + ) => SelectWithout) + | (( + ...columns: (Column | SQL)[] + ) => SelectWithout) + | (( + ...columns: + | [ + ( + aliases: this["_"]["selection"], + ) => ValueOrArray, + ] + | (Column | SQL | SQL.Aliased)[] + ) => SelectWithout); + + declare limit: ( + limit: number | Placeholder, + ) => SelectWithout; + + declare offset: ( + offset: number | Placeholder, + ) => SelectWithout; + + declare toSQL: () => Query; + + declare as: ( + alias: TAlias, + ) => SubqueryWithSelection; + + declare $dynamic: () => SelectDynamic; +} + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.ts#L803 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.ts#L903 + */ +export type SelectBase< + TTableName extends string | undefined, + TResultType extends "sync" | "async", + TRunResult, + TSelection extends ColumnsSelection, + TSelectMode extends SelectMode = "single", + TNullabilityMap extends Record< + string, + JoinNullability + > = TTableName extends string ? Record : {}, + TDynamic extends boolean = false, + TExcludedMethods extends string = never, + TResult = SelectResult[], + TSelectedFields extends ColumnsSelection = BuildSubquerySelection< + TSelection, + TNullabilityMap + >, +> = SelectQueryBuilderBase< + SelectHKT, + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode, + TNullabilityMap, + TDynamic, + TExcludedMethods, + // @ts-ignore + TResult, + TSelectedFields +> & + QueryPromise; + /** * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L31 * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L30 @@ -55,6 +234,62 @@ export type SelectJoinConfig = { joinType: JoinType; }; +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L38 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L30 + */ +export type BuildAliasTable< + tableOrView extends Table | View, + alias extends string, +> = tableOrView extends Table + ? TableWithColumns< + UpdateTableConfig< + tableOrView["_"]["config"], + { + name: alias; + columns: MapColumnsToTableAlias; + } + > + > + : tableOrView extends View + ? ViewWithSelection< + alias, + tableOrView["_"]["existing"], + MapColumnsToTableAlias + > + : never; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L52 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L52 + */ +export type SelectConfig = { + withList?: Subquery[]; + fields: Record; + fieldsFlat?: SelectedFieldsOrdered; + where?: SQL; + having?: SQL; + table: Table | Subquery | View | SQL; + limit?: number | Placeholder; + offset?: number | Placeholder; + joins?: SelectJoinConfig[]; + orderBy?: (Column | SQL | SQL.Aliased)[]; + groupBy?: (Column | SQL | SQL.Aliased)[]; + distinct?: boolean; + setOperators: { + rightSelect: TypedQueryBuilder; + type: SetOperator; + isAll: boolean; + orderBy?: (Column | SQL | SQL.Aliased)[]; + limit?: number | Placeholder; + offset?: number | Placeholder; + }[]; +}; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L75 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L82 + */ export type Join< T extends AnySelectQueryBuilder, TDynamic extends boolean, @@ -99,6 +334,10 @@ export type Join< > : never; +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L106 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L111 + */ export type JoinFn< T extends AnySelectQueryBuilder, TDynamic extends boolean, @@ -125,61 +364,9 @@ type MapColumnsToTableAlias< } & {}; /** - * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L38 - * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L30 - */ -export type BuildAliasTable< - tableOrView extends Table | View, - alias extends string, -> = tableOrView extends Table - ? TableWithColumns< - UpdateTableConfig< - tableOrView["_"]["config"], - { - name: alias; - columns: MapColumnsToTableAlias; - } - > - > - : tableOrView extends View - ? ViewWithSelection< - alias, - tableOrView["_"]["existing"], - MapColumnsToTableAlias - > - : never; - -/** - * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L52 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L124 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L146 */ -export type SelectConfig = { - withList?: Subquery[]; - fields: Record; - fieldsFlat?: SelectedFieldsOrdered; - where?: SQL; - having?: SQL; - table: Table | Subquery | View | SQL; - limit?: number | Placeholder; - offset?: number | Placeholder; - joins?: SelectJoinConfig[]; - orderBy?: (Column | SQL | SQL.Aliased)[]; - groupBy?: (Column | SQL | SQL.Aliased)[]; - distinct?: boolean; - setOperators: { - rightSelect: TypedQueryBuilder; - type: SetOperator; - isAll: boolean; - orderBy?: (Column | SQL | SQL.Aliased)[]; - limit?: number | Placeholder; - offset?: number | Placeholder; - }[]; -}; - -/** - * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L122 - */ -export type SelectedFieldsOrdered = SelectFieldsOrderedBase; - export type SelectHKTBase = { tableName: string | undefined; resultType: "sync" | "async"; @@ -194,21 +381,10 @@ export type SelectHKTBase = { _type: unknown; }; -export interface SelectHKT extends SelectHKTBase { - _type: SelectBase< - this["tableName"], - this["resultType"], - this["runResult"], - Assume, - this["selectMode"], - Assume>, - this["dynamic"], - this["excludedMethods"], - Assume, - Assume - >; -} - +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L138 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L158 + */ export type SelectKind< T extends SelectHKTBase, TTableName extends string | undefined, @@ -234,6 +410,10 @@ export type SelectKind< selectedFields: TSelectedFields; })["_type"]; +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L163 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L179 + */ export interface SelectQueryBuilderHKT extends SelectHKTBase { _type: SelectQueryBuilderBase< SelectQueryBuilderHKT, @@ -251,141 +431,37 @@ export interface SelectQueryBuilderHKT extends SelectHKTBase { } /** - * Partial implementation of the select query builder - * - * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.ts#L126 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L179 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L193 */ -export type SelectQueryBuilderBase< - THKT extends SelectHKTBase, - TTableName extends string | undefined, - TResultType extends "sync" | "async", - TRunResult, - TSelection extends ColumnsSelection, - TSelectMode extends SelectMode, - TNullabilityMap extends Record< - string, - JoinNullability - > = TTableName extends string ? Record : {}, - TDynamic extends boolean = false, - TExcludedMethods extends string = never, - TResult extends any[] = SelectResult< - TSelection, - TSelectMode, - TNullabilityMap - >[], - TSelectedFields extends ColumnsSelection = BuildSubquerySelection< - TSelection, - TNullabilityMap - >, -> = TypedQueryBuilder & { - [entityKind]: string; - _: { - readonly hkt: THKT; - readonly tableName: TTableName; - readonly resultType: TResultType; - readonly runResult: TRunResult; - readonly selection: TSelection; - readonly selectMode: TSelectMode; - readonly nullabilityMap: TNullabilityMap; - readonly dynamic: TDynamic; - readonly excludedMethods: TExcludedMethods; - readonly result: TResult; - readonly selectedFields: TSelectedFields; - }; - - leftJoin: JoinFn< - SelectQueryBuilderBase< - THKT, - TTableName, - TResultType, - TRunResult, - TSelection, - TSelectMode, - TNullabilityMap, - TDynamic, - TExcludedMethods, - TResult, - TSelectedFields - >, - TDynamic, - "left" - >; - - rightJoin: JoinFn< - SelectQueryBuilderBase< - THKT, - TTableName, - TResultType, - TRunResult, - TSelection, - TSelectMode, - TNullabilityMap, - TDynamic, - TExcludedMethods, - TResult, - TSelectedFields - >, - TDynamic, - "right" - >; - - innerJoin: JoinFn< - SelectQueryBuilderBase< - THKT, - TTableName, - TResultType, - TRunResult, - TSelection, - TSelectMode, - TNullabilityMap, - TDynamic, - TExcludedMethods, - TResult, - TSelectedFields - >, - TDynamic, - "inner" - >; - - fullJoin: JoinFn< - SelectQueryBuilderBase< - THKT, - TTableName, - TResultType, - TRunResult, - TSelection, - TSelectMode, - TNullabilityMap, - TDynamic, - TExcludedMethods, - TResult, - TSelectedFields - >, - TDynamic, - "full" +export interface SelectHKT extends SelectHKTBase { + _type: SelectBase< + this["tableName"], + this["resultType"], + this["runResult"], + Assume, + this["selectMode"], + Assume>, + this["dynamic"], + this["excludedMethods"], + Assume, + Assume >; +} - where: ( - where: ((aliases: TSelection) => SQL | undefined) | SQL | undefined, - ) => SelectWithout< - SelectQueryBuilderBase< - THKT, - TTableName, - TResultType, - TRunResult, - TSelection, - TSelectMode, - TNullabilityMap, - TDynamic, - TExcludedMethods, - TResult, - TSelectedFields - >, - TDynamic, - "where" - >; -}; +export type SetOperatorExcludedMethods = + | "leftJoin" + | "rightJoin" + | "innerJoin" + | "fullJoin" + | "where" + | "having" + | "groupBy"; +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L204 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L206 + */ export type CreateSelectFromBuilderMode< TBuilderMode extends "db" | "qb", TTableName extends string | undefined, @@ -404,20 +480,10 @@ export type CreateSelectFromBuilderMode< TSelectMode >; -export type AnySelectQueryBuilder = SelectQueryBuilderBase< - any, - any, - any, - any, - any, - any, - any, - any, - any, - any, - any ->; - +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/query-builders/select.types.ts#L227 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/query-builders/select.types.ts#L224 + */ type SelectWithout< T extends AnySelectQueryBuilder, TDynamic extends boolean, @@ -442,7 +508,47 @@ type SelectWithout< TResetExcluded extends true ? K : T["_"]["excludedMethods"] | K >; -export type SelectBase< +export type SelectDynamic = SelectKind< + T["_"]["hkt"], + T["_"]["tableName"], + T["_"]["resultType"], + T["_"]["runResult"], + T["_"]["selection"], + T["_"]["selectMode"], + T["_"]["nullabilityMap"], + true, + never, + T["_"]["result"], + T["_"]["selectedFields"] +>; + +export type AnySelectQueryBuilder = SelectQueryBuilderBase< + any, + any, + any, + any, + any, + any, + any, + any, + any, + any, + any +>; + +export type AnySetOperatorInterface = SetOperatorInterface< + any, + any, + any, + any, + any, + any, + any, + any, + any +>; + +export interface SetOperatorInterface< TTableName extends string | undefined, TResultType extends "sync" | "async", TRunResult, @@ -454,23 +560,182 @@ export type SelectBase< > = TTableName extends string ? Record : {}, TDynamic extends boolean = false, TExcludedMethods extends string = never, - TResult = SelectResult[], + TResult extends any[] = SelectResult< + TSelection, + TSelectMode, + TNullabilityMap + >[], TSelectedFields extends ColumnsSelection = BuildSubquerySelection< TSelection, TNullabilityMap >, -> = SelectQueryBuilderBase< - SelectHKT, - TTableName, - TResultType, - TRunResult, - TSelection, - TSelectMode, - TNullabilityMap, - TDynamic, - TExcludedMethods, - // @ts-ignore +> { + _: { + readonly hkt: SelectHKTBase; + readonly tableName: TTableName; + readonly resultType: TResultType; + readonly runResult: TRunResult; + readonly selection: TSelection; + readonly selectMode: TSelectMode; + readonly nullabilityMap: TNullabilityMap; + readonly dynamic: TDynamic; + readonly excludedMethods: TExcludedMethods; + readonly result: TResult; + readonly selectedFields: TSelectedFields; + }; +} + +export type SetOperatorWithResult = SetOperatorInterface< + any, + any, + any, + any, + any, + any, + any, + any, TResult, - TSelectedFields -> & - QueryPromise; + any +>; + +export type SetOperatorRightSelect< + TValue extends SetOperatorWithResult, + TResult extends any[], +> = TValue extends SetOperatorInterface< + any, + any, + any, + any, + any, + any, + any, + any, + infer TValueResult, + any +> + ? ValidateShape< + TValueResult[number], + TResult[number], + TypedQueryBuilder + > + : TValue; + +export type SetOperatorRestSelect< + TValue extends readonly SetOperatorWithResult[], + TResult extends any[], +> = TValue extends [infer First, ...infer Rest] + ? First extends SetOperatorInterface< + any, + any, + any, + any, + any, + any, + any, + any, + infer TValueResult, + any + > + ? Rest extends AnySetOperatorInterface[] + ? [ + ValidateShape< + TValueResult[number], + TResult[number], + TypedQueryBuilder + >, + ...SetOperatorRestSelect, + ] + : ValidateShape< + TValueResult[number], + TResult[number], + TypedQueryBuilder[] + > + : never + : TValue; + +export type CreateSetOperatorFn = < + TTableName extends string | undefined, + TResultType extends "sync" | "async", + TRunResult, + TSelection extends ColumnsSelection, + TValue extends SetOperatorWithResult, + TRest extends SetOperatorWithResult[], + TSelectMode extends SelectMode = "single", + TNullabilityMap extends Record< + string, + JoinNullability + > = TTableName extends string ? Record : {}, + TDynamic extends boolean = false, + TExcludedMethods extends string = never, + TResult extends any[] = SelectResult< + TSelection, + TSelectMode, + TNullabilityMap + >[], + TSelectedFields extends ColumnsSelection = BuildSubquerySelection< + TSelection, + TNullabilityMap + >, +>( + leftSelect: SetOperatorInterface< + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode, + TNullabilityMap, + TDynamic, + TExcludedMethods, + TResult, + TSelectedFields + >, + rightSelect: SetOperatorRightSelect, + ...restSelects: SetOperatorRestSelect +) => SelectWithout< + SelectBase< + TTableName, + TResultType, + TRunResult, + TSelection, + TSelectMode, + TNullabilityMap, + TDynamic, + TExcludedMethods, + TResult, + TSelectedFields + >, + false, + SetOperatorExcludedMethods, + true +>; + +export type GetSetOperators = { + union: CreateSetOperatorFn; + intersect: CreateSetOperatorFn; + except: CreateSetOperatorFn; + unionAll: CreateSetOperatorFn; +}; + +export type SubqueryWithSelection< + TSelection extends ColumnsSelection, + TAlias extends string, +> = Subquery> & + AddAliasToSelection; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/table.ts#L49 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/table.ts#L43 + */ +export type TableWithColumns = Table & { + [key in keyof T["columns"]]: T["columns"][key]; +}; + +/** + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/view.ts#L154 + * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/view.ts#L305 + */ +export type ViewWithSelection< + TName extends string, + TExisting extends boolean, + TSelection extends ColumnsSelection, +> = View & TSelection; diff --git a/packages/core/src/drizzle/table.ts b/packages/core/src/drizzle/table.ts index 4741b0ab1..2fc307f09 100644 --- a/packages/core/src/drizzle/table.ts +++ b/packages/core/src/drizzle/table.ts @@ -6,32 +6,8 @@ import type { ScalarColumn, } from "@/schema/common.js"; import type { InferColumnType } from "@/schema/infer.js"; -import type { - BuildColumns, - ColumnBuilderBase, - ColumnsSelection, - Table, - TableConfig, - View, -} from "drizzle-orm"; - -/** - * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/table.ts#L49 - * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/table.ts#L43 - */ -export type TableWithColumns = Table & { - [key in keyof T["columns"]]: T["columns"][key]; -}; - -/** - * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/sqlite-core/view.ts#L154 - * https://github.com/drizzle-team/drizzle-orm/blob/main/drizzle-orm/src/pg-core/view.ts#L305 - */ -export type ViewWithSelection< - TName extends string, - TExisting extends boolean, - TSelection extends ColumnsSelection, -> = View & TSelection; +import type { BuildColumns, ColumnBuilderBase } from "drizzle-orm"; +import type { TableWithColumns } from "./select.js"; /** * Performs type transformation between Ponder and Drizzle column representation. From acf23096a0e009b3306035a987ee9be72100cd1a Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 27 Jun 2024 12:02:58 -0400 Subject: [PATCH 057/122] pair programming with kevin --- packages/core/src/build/plugin.ts | 4 +- packages/core/src/build/service.ts | 160 ++++++++++++++++++++++----- packages/core/src/drizzle/runtime.ts | 2 - 3 files changed, 135 insertions(+), 31 deletions(-) diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 68ac18b9a..98f2ab492 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -29,8 +29,6 @@ export function replaceStateless(code: string, regex: RegExp, shim: string) { } export const vitePluginPonder = (): Plugin => { - const databasekind = "sqlite"; - return { name: "ponder", load: (id) => { @@ -40,7 +38,7 @@ import { convertToDrizzleTable } from "@ponder/core"; let drizzleTables = Object.fromEntries( Object.entries(schema).map(([tableName, table]) => [ tableName, - convertToDrizzleTable(tableName, table.table, "${databasekind}"), + convertToDrizzleTable(tableName, table.table, databaseConfig), ]), ); export * from "@ponder/core/virtual"; diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 53c3f208f..c17e54b0a 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -32,11 +32,15 @@ export type Service = { // static common: Common; srcRegex: RegExp; + serverRegex: RegExp; // vite viteDevServer: ViteDevServer; viteNodeServer: ViteNodeServer; viteNodeRunner: ViteNodeRunner; + + // state + databaseConfig?: DatabaseConfig; }; export type Build = { @@ -52,7 +56,9 @@ export type Build = { graphQLSchema: GraphQLSchema; // Indexing functions indexingFunctions: IndexingFunctions; - // Server +}; + +export type ServerBuild = { app?: Hono; }; @@ -60,6 +66,10 @@ export type BuildResult = | { status: "success"; build: Build } | { status: "error"; error: Error }; +export type ServerBuildResult = + | { status: "success"; build: ServerBuild } + | { status: "error"; error: Error }; + type RawBuild = { config: { config: Config; contentHash: string }; schema: { schema: Schema; contentHash: string }; @@ -67,7 +77,6 @@ type RawBuild = { indexingFunctions: RawIndexingFunctions; contentHash: string; }; - server: { app?: Hono }; }; export const create = async ({ @@ -84,6 +93,20 @@ export const create = async ({ .replace(escapeRegex, "\\$&"); const srcRegex = new RegExp(`^${escapedSrcDir}/.*\\.(ts|js)$`); + // TODO(kyle) handle js? + const escapedServerFile = common.options.serverFile + // If on Windows, use a POSIX path for this regex. + .replace(/\\/g, "/") + // Escape special characters in the path. + .replace(escapeRegex, "\\$&"); + const serverRegex = new RegExp(`^${escapedServerFile}$`); + + const service = { + common, + srcRegex, + serverRegex, + } as Service; + const viteLogger = { warnedMessages: new Set(), loggedErrors: new WeakSet(), @@ -115,7 +138,10 @@ export const create = async ({ publicDir: false, customLogger: viteLogger, server: { hmr: false }, - plugins: [viteTsconfigPathsPlugin(), vitePluginPonder()], + plugins: [ + viteTsconfigPathsPlugin(), + vitePluginPonder(() => service.databaseConfig), + ], }); // This is Vite boilerplate (initializes the Rollup container). @@ -132,13 +158,11 @@ export const create = async ({ resolveId: (id, importer) => viteNodeServer.resolveId(id, importer, "ssr"), }); - return { - common, - srcRegex, - viteDevServer, - viteNodeServer, - viteNodeRunner, - }; + service.viteDevServer = viteDevServer; + service.viteNodeServer = viteNodeServer; + service.viteNodeRunner = viteNodeRunner; + + return service; }; /** @@ -158,12 +182,11 @@ export const start = async ( ): Promise => { const { common } = buildService; - const [configResult, schemaResult, indexingFunctionsResult, serverResult] = + const [configResult, schemaResult, indexingFunctionsResult] = await Promise.all([ executeConfig(buildService), executeSchema(buildService), executeIndexingFunctions(buildService), - executeServer(buildService), ]); if (configResult.status === "error") { @@ -175,15 +198,11 @@ export const start = async ( if (indexingFunctionsResult.status === "error") { return { status: "error", error: indexingFunctionsResult.error }; } - if (serverResult.status === "error") { - return { status: "error", error: serverResult.error }; - } const rawBuild: RawBuild = { config: configResult, schema: schemaResult, indexingFunctions: indexingFunctionsResult, - server: serverResult, }; const buildResult = await validateAndBuild(buildService, rawBuild); @@ -279,15 +298,6 @@ export const start = async ( rawBuild.indexingFunctions = result; } - if (hasSrcUpdate) { - const result = await executeServer(buildService); - if (result.status === "error") { - onBuild({ status: "error", error: result.error }); - return; - } - rawBuild.server = result; - } - const buildResult = await validateAndBuild(buildService, rawBuild); onBuild(buildResult); }; @@ -298,6 +308,104 @@ export const start = async ( return buildResult; }; +export const serverStart = async ( + buildService: Service, + { + watch, + onBuild, + }: + | { watch: true; onBuild: (buildResult: ServerBuildResult) => void } + | { watch: false; onBuild?: never }, + databaseConfig: DatabaseConfig, +): Promise => { + buildService.databaseConfig = databaseConfig; + console.log({ databaseConfig }); + const { common } = buildService; + + const serverResult = await executeServer(buildService); + + if (serverResult.status === "error") { + return { status: "error", error: serverResult.error }; + } + + const serverBuild: ServerBuild = { + app: serverResult.app, + }; + + // If watch is false (`ponder start` or `ponder serve`), + // don't register any event handlers on the watcher. + if (watch) { + // Define the directories and files to ignore + const ignoredDirs = [common.options.generatedDir, common.options.ponderDir]; + const ignoredFiles = [ + path.join(common.options.rootDir, "ponder-env.d.ts"), + path.join(common.options.rootDir, ".env.local"), + ]; + + const isFileIgnored = (filePath: string) => { + const isInIgnoredDir = ignoredDirs.some((dir) => { + const rel = path.relative(dir, filePath); + return !rel.startsWith("..") && !path.isAbsolute(rel); + }); + + const isIgnoredFile = ignoredFiles.includes(filePath); + return isInIgnoredDir || isIgnoredFile; + }; + + const onFileChange = async (_file: string) => { + if (isFileIgnored(_file)) return; + + // Note that `toFilePath` always returns a POSIX path, even if you pass a Windows path. + const file = toFilePath( + normalizeModuleId(_file), + common.options.rootDir, + ).path; + + // Invalidate all modules that depend on the updated files. + // Note that `invalidateDepTree` accepts and returns POSIX paths, even on Windows. + const invalidated = [ + ...buildService.viteNodeRunner.moduleCache.invalidateDepTree([file]), + ]; + + // If no files were invalidated, no need to reload. + if (invalidated.length === 0) return; + + // Note that the paths in `invalidated` are POSIX, so we need to + // convert the paths in `options` to POSIX for this comparison. + // The `srcDir` regex is already converted to POSIX. + const hasServerUpdate = invalidated.some((file) => + buildService.serverRegex.test(file), + ); + + // This branch could trigger if you change a `note.txt` file within `src/`. + // Note: We could probably do a better job filtering out files in `isFileIgnored`. + if (!hasServerUpdate) { + return; + } + + common.logger.info({ + service: "build", + msg: `Hot reload ${invalidated + .map((f) => `'${path.relative(common.options.rootDir, f)}'`) + .join(", ")}`, + }); + + const result = await executeServer(buildService); + if (result.status === "error") { + onBuild({ status: "error", error: result.error }); + return; + } + serverBuild.app = result.app; + + onBuild({ status: "success", build: serverBuild }); + }; + + buildService.viteDevServer.watcher.on("change", onFileChange); + } + + return { status: "success", build: serverBuild }; +}; + export const kill = async (buildService: Service): Promise => { await buildService.viteDevServer?.close(); buildService.common.logger.debug({ @@ -378,6 +486,7 @@ const executeIndexingFunctions = async ( .join(buildService.common.options.srcDir, "**/*.{js,mjs,ts,mts}") .replace(/\\/g, "/"); // TODO(kyle) ignore server file + const files = glob.sync(pattern); const executeResults = await Promise.all( @@ -531,7 +640,6 @@ const validateAndBuild = async ( graphQLSchema, indexingFunctions: buildConfigAndIndexingFunctionsResult.indexingFunctions, - app: rawBuild.server.app, }, }; }; diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index 2356a6f38..e82c6c792 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -52,8 +52,6 @@ export const createDrizzleDb = ( } }; -// TODO: handle schemas - export const convertToDrizzleTable = ( tableName: string, table: PonderTable, From a6011227b2b1290201506402e92be28f0a4b94ef Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 27 Jun 2024 12:06:23 -0400 Subject: [PATCH 058/122] revert build service changes --- packages/core/src/build/service.ts | 160 +++++------------------------ 1 file changed, 26 insertions(+), 134 deletions(-) diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index c17e54b0a..53c3f208f 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -32,15 +32,11 @@ export type Service = { // static common: Common; srcRegex: RegExp; - serverRegex: RegExp; // vite viteDevServer: ViteDevServer; viteNodeServer: ViteNodeServer; viteNodeRunner: ViteNodeRunner; - - // state - databaseConfig?: DatabaseConfig; }; export type Build = { @@ -56,9 +52,7 @@ export type Build = { graphQLSchema: GraphQLSchema; // Indexing functions indexingFunctions: IndexingFunctions; -}; - -export type ServerBuild = { + // Server app?: Hono; }; @@ -66,10 +60,6 @@ export type BuildResult = | { status: "success"; build: Build } | { status: "error"; error: Error }; -export type ServerBuildResult = - | { status: "success"; build: ServerBuild } - | { status: "error"; error: Error }; - type RawBuild = { config: { config: Config; contentHash: string }; schema: { schema: Schema; contentHash: string }; @@ -77,6 +67,7 @@ type RawBuild = { indexingFunctions: RawIndexingFunctions; contentHash: string; }; + server: { app?: Hono }; }; export const create = async ({ @@ -93,20 +84,6 @@ export const create = async ({ .replace(escapeRegex, "\\$&"); const srcRegex = new RegExp(`^${escapedSrcDir}/.*\\.(ts|js)$`); - // TODO(kyle) handle js? - const escapedServerFile = common.options.serverFile - // If on Windows, use a POSIX path for this regex. - .replace(/\\/g, "/") - // Escape special characters in the path. - .replace(escapeRegex, "\\$&"); - const serverRegex = new RegExp(`^${escapedServerFile}$`); - - const service = { - common, - srcRegex, - serverRegex, - } as Service; - const viteLogger = { warnedMessages: new Set(), loggedErrors: new WeakSet(), @@ -138,10 +115,7 @@ export const create = async ({ publicDir: false, customLogger: viteLogger, server: { hmr: false }, - plugins: [ - viteTsconfigPathsPlugin(), - vitePluginPonder(() => service.databaseConfig), - ], + plugins: [viteTsconfigPathsPlugin(), vitePluginPonder()], }); // This is Vite boilerplate (initializes the Rollup container). @@ -158,11 +132,13 @@ export const create = async ({ resolveId: (id, importer) => viteNodeServer.resolveId(id, importer, "ssr"), }); - service.viteDevServer = viteDevServer; - service.viteNodeServer = viteNodeServer; - service.viteNodeRunner = viteNodeRunner; - - return service; + return { + common, + srcRegex, + viteDevServer, + viteNodeServer, + viteNodeRunner, + }; }; /** @@ -182,11 +158,12 @@ export const start = async ( ): Promise => { const { common } = buildService; - const [configResult, schemaResult, indexingFunctionsResult] = + const [configResult, schemaResult, indexingFunctionsResult, serverResult] = await Promise.all([ executeConfig(buildService), executeSchema(buildService), executeIndexingFunctions(buildService), + executeServer(buildService), ]); if (configResult.status === "error") { @@ -198,11 +175,15 @@ export const start = async ( if (indexingFunctionsResult.status === "error") { return { status: "error", error: indexingFunctionsResult.error }; } + if (serverResult.status === "error") { + return { status: "error", error: serverResult.error }; + } const rawBuild: RawBuild = { config: configResult, schema: schemaResult, indexingFunctions: indexingFunctionsResult, + server: serverResult, }; const buildResult = await validateAndBuild(buildService, rawBuild); @@ -298,6 +279,15 @@ export const start = async ( rawBuild.indexingFunctions = result; } + if (hasSrcUpdate) { + const result = await executeServer(buildService); + if (result.status === "error") { + onBuild({ status: "error", error: result.error }); + return; + } + rawBuild.server = result; + } + const buildResult = await validateAndBuild(buildService, rawBuild); onBuild(buildResult); }; @@ -308,104 +298,6 @@ export const start = async ( return buildResult; }; -export const serverStart = async ( - buildService: Service, - { - watch, - onBuild, - }: - | { watch: true; onBuild: (buildResult: ServerBuildResult) => void } - | { watch: false; onBuild?: never }, - databaseConfig: DatabaseConfig, -): Promise => { - buildService.databaseConfig = databaseConfig; - console.log({ databaseConfig }); - const { common } = buildService; - - const serverResult = await executeServer(buildService); - - if (serverResult.status === "error") { - return { status: "error", error: serverResult.error }; - } - - const serverBuild: ServerBuild = { - app: serverResult.app, - }; - - // If watch is false (`ponder start` or `ponder serve`), - // don't register any event handlers on the watcher. - if (watch) { - // Define the directories and files to ignore - const ignoredDirs = [common.options.generatedDir, common.options.ponderDir]; - const ignoredFiles = [ - path.join(common.options.rootDir, "ponder-env.d.ts"), - path.join(common.options.rootDir, ".env.local"), - ]; - - const isFileIgnored = (filePath: string) => { - const isInIgnoredDir = ignoredDirs.some((dir) => { - const rel = path.relative(dir, filePath); - return !rel.startsWith("..") && !path.isAbsolute(rel); - }); - - const isIgnoredFile = ignoredFiles.includes(filePath); - return isInIgnoredDir || isIgnoredFile; - }; - - const onFileChange = async (_file: string) => { - if (isFileIgnored(_file)) return; - - // Note that `toFilePath` always returns a POSIX path, even if you pass a Windows path. - const file = toFilePath( - normalizeModuleId(_file), - common.options.rootDir, - ).path; - - // Invalidate all modules that depend on the updated files. - // Note that `invalidateDepTree` accepts and returns POSIX paths, even on Windows. - const invalidated = [ - ...buildService.viteNodeRunner.moduleCache.invalidateDepTree([file]), - ]; - - // If no files were invalidated, no need to reload. - if (invalidated.length === 0) return; - - // Note that the paths in `invalidated` are POSIX, so we need to - // convert the paths in `options` to POSIX for this comparison. - // The `srcDir` regex is already converted to POSIX. - const hasServerUpdate = invalidated.some((file) => - buildService.serverRegex.test(file), - ); - - // This branch could trigger if you change a `note.txt` file within `src/`. - // Note: We could probably do a better job filtering out files in `isFileIgnored`. - if (!hasServerUpdate) { - return; - } - - common.logger.info({ - service: "build", - msg: `Hot reload ${invalidated - .map((f) => `'${path.relative(common.options.rootDir, f)}'`) - .join(", ")}`, - }); - - const result = await executeServer(buildService); - if (result.status === "error") { - onBuild({ status: "error", error: result.error }); - return; - } - serverBuild.app = result.app; - - onBuild({ status: "success", build: serverBuild }); - }; - - buildService.viteDevServer.watcher.on("change", onFileChange); - } - - return { status: "success", build: serverBuild }; -}; - export const kill = async (buildService: Service): Promise => { await buildService.viteDevServer?.close(); buildService.common.logger.debug({ @@ -486,7 +378,6 @@ const executeIndexingFunctions = async ( .join(buildService.common.options.srcDir, "**/*.{js,mjs,ts,mts}") .replace(/\\/g, "/"); // TODO(kyle) ignore server file - const files = glob.sync(pattern); const executeResults = await Promise.all( @@ -640,6 +531,7 @@ const validateAndBuild = async ( graphQLSchema, indexingFunctions: buildConfigAndIndexingFunctionsResult.indexingFunctions, + app: rawBuild.server.app, }, }; }; From f38aadfd1a4ad383561df0ac51ef9d49c2ee06cf Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 27 Jun 2024 12:24:51 -0400 Subject: [PATCH 059/122] use schemas in drizzle tables --- packages/core/src/bin/commands/serve.ts | 5 +- packages/core/src/bin/utils/run.ts | 3 +- packages/core/src/drizzle/runtime.test.ts | 47 +++++++++++++++---- packages/core/src/drizzle/runtime.ts | 44 ++++++++++++----- .../{table.test.ts => table.test-d.ts} | 24 ++++++++-- 5 files changed, 93 insertions(+), 30 deletions(-) rename packages/core/src/drizzle/{table.test.ts => table.test-d.ts} (53%) diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index d09f62ff9..c014c1c6e 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -8,7 +8,6 @@ import { PostgresDatabaseService } from "@/database/postgres/service.js"; import type { NamespaceInfo } from "@/database/service.js"; import { getReadonlyStore } from "@/indexing-store/readonly.js"; import { createServer } from "@/server/service.js"; -import type { RawBuilder } from "kysely"; import type { CliOptions } from "../ponder.js"; import { setupShutdown } from "../utils/shutdown.js"; @@ -114,9 +113,7 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { app, readonlyStore, schema, - query: (query: RawBuilder) => { - return query.execute(database.readonlyDb.withSchema(userNamespace)); - }, + database: { kind: "postgres", pool: database.readonlyPool }, common, }); server.setHealthy(); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 73ca9814a..f753a2bc2 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -106,11 +106,10 @@ export async function run({ app: build.app, readonlyStore, schema, - // @ts-ignore database: database.kind === "sqlite" ? { kind: "sqlite", database: database.userDatabase } - : { kind: "postgres" }, + : { kind: "postgres", pool: database.readonlyPool }, common, }); diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index e127b76df..b1588943d 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -8,6 +8,7 @@ import { SqliteDatabaseService } from "@/database/sqlite/service.js"; import type { HistoricalStore } from "@/indexing-store/store.js"; import { createSchema } from "@/schema/schema.js"; import { beforeEach, expect, test } from "vitest"; +import type { DrizzleDb } from "./db.js"; import { convertToDrizzleTable, createDrizzleDb } from "./runtime.js"; beforeEach(setupCommon); @@ -18,12 +19,12 @@ const createDb = (database: DatabaseService) => { return createDrizzleDb({ kind: database.kind, database: database.userDatabase, - }) as any; + }) as unknown as DrizzleDb; } else { return createDrizzleDb({ - kind: "postgres", + kind: database.kind, pool: database.readonlyPool, - }) as any; + }) as unknown as DrizzleDb; } }; @@ -46,7 +47,13 @@ test("runtime select", async (context) => { const rows = await db .select() - .from(convertToDrizzleTable("table", schema.table.table, database.kind)); + .from( + convertToDrizzleTable( + "table", + schema.table.table, + context.databaseConfig, + ), + ); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: "kyle" }); @@ -73,7 +80,13 @@ test("runtime hex", async (context) => { const rows = await db .select() - .from(convertToDrizzleTable("table", schema.table.table, "postgres")); + .from( + convertToDrizzleTable( + "table", + schema.table.table, + context.databaseConfig, + ), + ); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: "0x01" }); @@ -100,7 +113,13 @@ test("runtime bigint", async (context) => { const rows = await db .select() - .from(convertToDrizzleTable("table", schema.table.table, database.kind)); + .from( + convertToDrizzleTable( + "table", + schema.table.table, + context.databaseConfig, + ), + ); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: 1n }); @@ -136,7 +155,13 @@ test("runtime json", async (context) => { const rows = await db .select() - .from(convertToDrizzleTable("table", schema.table.table, database.kind)); + .from( + convertToDrizzleTable( + "table", + schema.table.table, + context.databaseConfig, + ), + ); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: "1", json: { prop: 52 } }); @@ -169,7 +194,13 @@ test("runtime enum", async (context) => { const rows = await db .select() - .from(convertToDrizzleTable("table", schema.table.table, database.kind)); + .from( + convertToDrizzleTable( + "table", + schema.table.table, + context.databaseConfig, + ), + ); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: "1", en: "hi" }); diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index e82c6c792..51110f220 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -1,3 +1,4 @@ +import type { DatabaseConfig } from "@/config/database.js"; import type { Table as PonderTable } from "@/schema/common.js"; import { isEnumColumn, @@ -11,7 +12,7 @@ import type { SqliteDatabase } from "@/utils/sqlite.js"; import type { Table } from "drizzle-orm"; import { drizzle as drizzleSQLite } from "drizzle-orm/better-sqlite3"; import { drizzle as drizzlePg } from "drizzle-orm/node-postgres"; -import { pgTable } from "drizzle-orm/pg-core"; +import { pgSchema, pgTable } from "drizzle-orm/pg-core"; import { doublePrecision as PgDoublePrecision, integer as PgInteger, @@ -55,39 +56,57 @@ export const createDrizzleDb = ( export const convertToDrizzleTable = ( tableName: string, table: PonderTable, - kind: "sqlite" | "postgres", + databaseConfig: DatabaseConfig, ): Table => { const columns = Object.entries(table).reduce<{ [columnName: string]: any }>( (acc, [columnName, column]) => { if (isMaterialColumn(column)) { if (isJSONColumn(column)) { - acc[columnName] = convertJsonColumn(columnName, kind); + acc[columnName] = convertJsonColumn(columnName, databaseConfig.kind); } else if (isEnumColumn(column)) { - acc[columnName] = convertEnumColumn(columnName, kind); + acc[columnName] = convertEnumColumn(columnName, databaseConfig.kind); } else if (isScalarColumn(column) || isReferenceColumn(column)) { switch (column[" scalar"]) { case "string": - acc[columnName] = convertStringColumn(columnName, kind); + acc[columnName] = convertStringColumn( + columnName, + databaseConfig.kind, + ); break; case "int": - acc[columnName] = convertIntColumn(columnName, kind); + acc[columnName] = convertIntColumn( + columnName, + databaseConfig.kind, + ); break; case "boolean": - acc[columnName] = convertBooleanColumn(columnName, kind); + acc[columnName] = convertBooleanColumn( + columnName, + databaseConfig.kind, + ); break; case "float": - acc[columnName] = convertFloatColumn(columnName, kind); + acc[columnName] = convertFloatColumn( + columnName, + databaseConfig.kind, + ); break; case "hex": - acc[columnName] = convertHexColumn(columnName, kind); + acc[columnName] = convertHexColumn( + columnName, + databaseConfig.kind, + ); break; case "bigint": - acc[columnName] = convertBigintColumn(columnName, kind); + acc[columnName] = convertBigintColumn( + columnName, + databaseConfig.kind, + ); break; } @@ -104,8 +123,9 @@ export const convertToDrizzleTable = ( {}, ); - if (kind === "postgres") { - return pgTable(tableName, columns); + if (databaseConfig.kind === "postgres") { + if (databaseConfig.schema === "public") return pgTable(tableName, columns); + return pgSchema(databaseConfig.schema).table(tableName, columns); } else { return sqliteTable(tableName, columns); } diff --git a/packages/core/src/drizzle/table.test.ts b/packages/core/src/drizzle/table.test-d.ts similarity index 53% rename from packages/core/src/drizzle/table.test.ts rename to packages/core/src/drizzle/table.test-d.ts index 181ad00c3..078e7b193 100644 --- a/packages/core/src/drizzle/table.test.ts +++ b/packages/core/src/drizzle/table.test-d.ts @@ -1,12 +1,20 @@ -import type { IdColumn, ScalarColumn } from "@/schema/common.js"; +import { createSchema } from "@/index.js"; import { expectTypeOf, test } from "vitest"; import type { DrizzleDb } from "./db.js"; import type { ConvertToDrizzleTable } from "./table.js"; test("select query promise", async () => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.string(), + name: p.int().optional(), + }), + })); + const table = {} as ConvertToDrizzleTable< "table", - { id: IdColumn<"string">; name: ScalarColumn<"int", true> } + (typeof schema)["table"]["table"], + typeof schema >; const result = await ({} as DrizzleDb).select({ id: table.id }).from(table); @@ -16,13 +24,21 @@ test("select query promise", async () => { }); test("select optional column", async () => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.string(), + name: p.int().optional(), + }), + })); + const table = {} as ConvertToDrizzleTable< "table", - { id: IdColumn<"string">; n: ScalarColumn<"int", true> } + (typeof schema)["table"]["table"], + typeof schema >; const result = await ({} as DrizzleDb).select().from(table); // ^? - expectTypeOf<{ id: string; n: number | null }[]>(result); + expectTypeOf<{ id: string; name: number | null }[]>(result); }); From 0fe58bdbd6a7e1b967c232bbaa05d1f1be5370fe Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Thu, 27 Jun 2024 12:57:41 -0400 Subject: [PATCH 060/122] plugin without serve fix --- packages/core/src/build/plugin.ts | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 98f2ab492..6ea651f63 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -34,7 +34,37 @@ export const vitePluginPonder = (): Plugin => { load: (id) => { if (id === "ponder:db") { return `import schema from "ponder.schema"; +import config from "ponder.config"; import { convertToDrizzleTable } from "@ponder/core"; +let databaseConfig = undefined +let envSchema = undefined; +let envPublishSchema = undefined; +if (process.env.RAILWAY_DEPLOYMENT_ID && process.env.RAILWAY_SERVICE_NAME) { + envSchema = \`\${process.env.RAILWAY_SERVICE_NAME}_\${process.env.RAILWAY_DEPLOYMENT_ID.slice( + 0, + 8, + )}\`; + envPublishSchema = "public"; +} else { + envSchema = "public"; +} +if (config.database?.kind) { + if (config.database.kind === "postgres") { + const schema = config.database.schema ?? envSchema; + const publishSchema = config.database.publishSchema ?? envPublishSchema; + databaseConfig = { kind: "postgres", schema, publishSchema }; + } else { + databaseConfig = { kind: "sqlite" }; + } +} else { + if (process.env.DATABASE_PRIVATE_URL || process.env.DATABASE_URL) { + const schema = envSchema; + const publishSchema = envPublishSchema; + databaseConfig = { kind: "postgres", schema, publishSchema }; + } else { + databaseConfig = { kind: "sqlite" }; + } +} let drizzleTables = Object.fromEntries( Object.entries(schema).map(([tableName, table]) => [ tableName, From 03e36b596aa585034fc872f5aed5d731a941e182 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 27 Jun 2024 14:40:41 -0400 Subject: [PATCH 061/122] type configuration --- examples/reference-erc20/ponder-env.d.ts | 6 ++-- packages/core/package.json | 2 +- packages/core/src/common/codegen.ts | 6 ++-- packages/core/src/drizzle/virtual.d.ts | 35 ------------------------ 4 files changed, 9 insertions(+), 40 deletions(-) delete mode 100644 packages/core/src/drizzle/virtual.d.ts diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index 1c6c13d45..2f78e46d0 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -3,8 +3,6 @@ // If this happens, please commit the changes. Do not manually edit this file. // See https://ponder.sh/docs/getting-started/installation#typescript for more information. -/// - declare module "@/generated" { import type { Virtual } from "@ponder/core"; @@ -43,3 +41,7 @@ declare module "ponder:db" { export = drizzleTables; } + +declare module "ponder:db" { + export * from "@ponder/core/drizzle"; +} diff --git a/packages/core/package.json b/packages/core/package.json index edf742ef2..03f95e214 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -27,7 +27,7 @@ "import": "./dist/index.js", "types": "./dist/index.d.ts" }, - "./virtual": { + "./drizzle": { "import": "./dist/drizzle/virtual.js", "types": "./dist/drizzle/virtual.d.ts" } diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index e2f9a2fc7..100a8347d 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -8,8 +8,6 @@ export const ponderEnv = `// This file enables type checking and editor autocomp // If this happens, please commit the changes. Do not manually edit this file. // See https://ponder.sh/docs/getting-started/installation#typescript for more information. -/// - declare module "@/generated" { import type { Virtual } from "@ponder/core"; @@ -48,6 +46,10 @@ declare module "ponder:db" { export = drizzleTables; } + +declare module "ponder:db" { + export * from "@ponder/core/drizzle"; +} `; export function runCodegen({ diff --git a/packages/core/src/drizzle/virtual.d.ts b/packages/core/src/drizzle/virtual.d.ts deleted file mode 100644 index 73d1bcef5..000000000 --- a/packages/core/src/drizzle/virtual.d.ts +++ /dev/null @@ -1,35 +0,0 @@ -declare module "ponder:db" { - type RuntimeConfig = typeof import("./virtual.js"); - - export const sql: RuntimeConfig["sql"]; - - export const eq: RuntimeConfig["eq"]; - export const gt: RuntimeConfig["gt"]; - export const gte: RuntimeConfig["gte"]; - export const lt: RuntimeConfig["lt"]; - export const lte: RuntimeConfig["lte"]; - export const ne: RuntimeConfig["ne"]; - export const isNull: RuntimeConfig["isNull"]; - export const isNotNull: RuntimeConfig["isNotNull"]; - export const inArray: RuntimeConfig["inArray"]; - export const notInArray: RuntimeConfig["notInArray"]; - export const exists: RuntimeConfig["exists"]; - export const notExists: RuntimeConfig["notExists"]; - export const between: RuntimeConfig["between"]; - export const notBetween: RuntimeConfig["notBetween"]; - export const like: RuntimeConfig["like"]; - export const notIlike: RuntimeConfig["notIlike"]; - export const not: RuntimeConfig["not"]; - export const asc: RuntimeConfig["asc"]; - export const desc: RuntimeConfig["desc"]; - export const and: RuntimeConfig["and"]; - export const or: RuntimeConfig["or"]; - export const count: RuntimeConfig["count"]; - export const countDistinct: RuntimeConfig["countDistinct"]; - export const avg: RuntimeConfig["avg"]; - export const avgDistinct: RuntimeConfig["avgDistinct"]; - export const sum: RuntimeConfig["sum"]; - export const sumDistinct: RuntimeConfig["sumDistinct"]; - export const max: RuntimeConfig["max"]; - export const min: RuntimeConfig["min"]; -} From 020f70e764e42e6b2d4e69fd29beb8b12c267307 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 27 Jun 2024 14:47:15 -0400 Subject: [PATCH 062/122] simplify order by and group by --- packages/core/src/drizzle/select.ts | 46 +++++------------------------ 1 file changed, 7 insertions(+), 39 deletions(-) diff --git a/packages/core/src/drizzle/select.ts b/packages/core/src/drizzle/select.ts index 76240ddb0..dc6a592d2 100644 --- a/packages/core/src/drizzle/select.ts +++ b/packages/core/src/drizzle/select.ts @@ -10,8 +10,6 @@ import type { TableConfig, UpdateTableConfig, ValidateShape, - ValueOrArray, - // ValueOrArray, . entityKind, } from "drizzle-orm"; import { TypedQueryBuilder } from "drizzle-orm/query-builders/query-builder"; @@ -131,43 +129,13 @@ export abstract class SelectQueryBuilderBase< | undefined, ) => SelectWithout; - declare groupBy: - | (( - builder: ( - aliases: this["_"]["selection"], - ) => ValueOrArray, - ) => SelectWithout) - | (( - ...columns: (Column | SQL)[] - ) => SelectWithout) - | (( - ...columns: - | [ - ( - aliases: this["_"]["selection"], - ) => ValueOrArray, - ] - | (Column | SQL | SQL.Aliased)[] - ) => SelectWithout); - - declare orderBy: - | (( - builder: ( - aliases: this["_"]["selection"], - ) => ValueOrArray, - ) => SelectWithout) - | (( - ...columns: (Column | SQL)[] - ) => SelectWithout) - | (( - ...columns: - | [ - ( - aliases: this["_"]["selection"], - ) => ValueOrArray, - ] - | (Column | SQL | SQL.Aliased)[] - ) => SelectWithout); + declare groupBy: ( + ...columns: (Column | SQL)[] + ) => SelectWithout; + + declare orderBy: ( + ...columns: (Column | SQL)[] + ) => SelectWithout; declare limit: ( limit: number | Placeholder, From ebb40760992c7956798b4a5b665927d578127d39 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 27 Jun 2024 16:05:16 -0400 Subject: [PATCH 063/122] join tests --- packages/core/src/drizzle/runtime.test.ts | 70 +++++++++++++++++++++-- packages/core/src/drizzle/table.test-d.ts | 47 +++++++++++++++ 2 files changed, 113 insertions(+), 4 deletions(-) diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index b1588943d..5a52f0bb0 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -10,6 +10,7 @@ import { createSchema } from "@/schema/schema.js"; import { beforeEach, expect, test } from "vitest"; import type { DrizzleDb } from "./db.js"; import { convertToDrizzleTable, createDrizzleDb } from "./runtime.js"; +import { eq } from "./virtual.js"; beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); @@ -61,7 +62,7 @@ test("runtime select", async (context) => { await cleanup(); }); -test("runtime hex", async (context) => { +test("select hex", async (context) => { const schema = createSchema((p) => ({ table: p.createTable({ id: p.hex(), @@ -94,7 +95,7 @@ test("runtime hex", async (context) => { await cleanup(); }); -test("runtime bigint", async (context) => { +test("select bigint", async (context) => { const schema = createSchema((p) => ({ table: p.createTable({ id: p.bigint(), @@ -127,7 +128,7 @@ test("runtime bigint", async (context) => { await cleanup(); }); -test("runtime json", async (context) => { +test("select json", async (context) => { const schema = createSchema((p) => ({ table: p.createTable({ id: p.string(), @@ -169,7 +170,7 @@ test("runtime json", async (context) => { await cleanup(); }); -test("runtime enum", async (context) => { +test("select enum", async (context) => { const schema = createSchema((p) => ({ en: p.createEnum(["hi", "low"]), table: p.createTable({ @@ -207,3 +208,64 @@ test("runtime enum", async (context) => { await cleanup(); }); + +test("select with join", async (context) => { + const schema = createSchema((p) => ({ + account: p.createTable({ + id: p.hex(), + name: p.string(), + age: p.int(), + }), + nft: p.createTable({ + id: p.bigint(), + owner: p.hex().references("account.id"), + }), + })); + + const { database, cleanup, indexingStore } = await setupDatabaseServices( + context, + { schema }, + ); + + const db = createDb(database); + + await indexingStore.create({ + tableName: "account", + id: "0x1", + data: { + name: "kyle", + age: 52, + }, + }); + await indexingStore.create({ + tableName: "nft", + id: 10n, + data: { owner: "0x1" }, + }); + await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); + + const account = convertToDrizzleTable( + "account", + schema.account.table, + context.databaseConfig, + ); + const nft = convertToDrizzleTable( + "nft", + schema.nft.table, + context.databaseConfig, + ); + + const rows = await db + .select() + .from(account) + // @ts-ignore + .fullJoin(nft, eq(account.id, nft.owner)); + + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + account: { id: "0x01", name: "kyle", age: 52 }, + nft: { id: 10n, owner: "0x01" }, + }); + + await cleanup(); +}); diff --git a/packages/core/src/drizzle/table.test-d.ts b/packages/core/src/drizzle/table.test-d.ts index 078e7b193..83e8f4774 100644 --- a/packages/core/src/drizzle/table.test-d.ts +++ b/packages/core/src/drizzle/table.test-d.ts @@ -1,7 +1,9 @@ import { createSchema } from "@/index.js"; +import type { Hex } from "viem"; import { expectTypeOf, test } from "vitest"; import type { DrizzleDb } from "./db.js"; import type { ConvertToDrizzleTable } from "./table.js"; +import { eq } from "./virtual.js"; test("select query promise", async () => { const schema = createSchema((p) => ({ @@ -42,3 +44,48 @@ test("select optional column", async () => { expectTypeOf<{ id: string; name: number | null }[]>(result); }); + +test("select join", async () => { + const schema = createSchema((p) => ({ + account: p.createTable({ + id: p.hex(), + name: p.string(), + age: p.int(), + }), + nft: p.createTable({ + id: p.bigint(), + owner: p.hex().references("account.id"), + }), + })); + + const account = {} as ConvertToDrizzleTable< + "account", + (typeof schema)["account"]["table"], + typeof schema + >; + const nft = {} as ConvertToDrizzleTable< + "nft", + (typeof schema)["nft"]["table"], + typeof schema + >; + + const result = await ({} as DrizzleDb) + // ^? + .select() + .from(account) + .fullJoin(nft, eq(account.id, nft.owner)); + + expectTypeOf< + { + account: { + id: Hex; + name: string; + age: number; + } | null; + nft: { + id: bigint; + owner: Hex; + } | null; + }[] + >(result); +}); From 9afab482fe30eea06b33e9b089b6a54ad643b1b5 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 10 Jul 2024 12:32:18 -0400 Subject: [PATCH 064/122] status endpoint --- packages/core/src/bin/commands/serve.ts | 17 ++++- packages/core/src/bin/utils/run.ts | 69 ++++++++++++++++--- .../core/src/database/postgres/migrations.ts | 4 ++ .../src/database/postgres/service.test.ts | 17 +++++ .../core/src/database/postgres/service.ts | 19 ++++- .../core/src/database/sqlite/migrations.ts | 4 ++ .../core/src/database/sqlite/service.test.ts | 7 ++ packages/core/src/database/sqlite/service.ts | 15 ++++ .../core/src/indexing-store/metadata.test.ts | 59 ++++++++++++++++ packages/core/src/indexing-store/metadata.ts | 47 +++++++++++++ packages/core/src/indexing-store/store.ts | 13 ++++ packages/core/src/server/service.test.ts | 35 ++++++++-- packages/core/src/server/service.ts | 32 ++++++--- packages/core/src/sync-realtime/index.ts | 3 +- packages/core/src/sync-realtime/service.ts | 20 ++++++ packages/core/src/sync/index.ts | 2 + packages/core/src/sync/service.ts | 35 ++++++++++ 17 files changed, 369 insertions(+), 29 deletions(-) create mode 100644 packages/core/src/indexing-store/metadata.test.ts create mode 100644 packages/core/src/indexing-store/metadata.ts diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index 068598910..5513bd8bf 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -6,6 +6,7 @@ import { buildOptions } from "@/common/options.js"; import { buildPayload, createTelemetry } from "@/common/telemetry.js"; import { PostgresDatabaseService } from "@/database/postgres/service.js"; import type { NamespaceInfo } from "@/database/service.js"; +import { getMetadataStore } from "@/indexing-store/metadata.js"; import { getReadonlyStore } from "@/indexing-store/readonly.js"; import { createServer } from "@/server/service.js"; import type { CliOptions } from "../ponder.js"; @@ -110,8 +111,20 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { common, }); - const server = await createServer({ graphqlSchema, common, readonlyStore }); - server.setHealthy(); + const metadataStore = getMetadataStore({ + encoding: database.kind, + namespaceInfo: { + userNamespace: databaseConfig.publishSchema, + } as unknown as NamespaceInfo, + db: database.readonlyDb, + }); + + const server = await createServer({ + graphqlSchema, + common, + readonlyStore, + metadataStore, + }); cleanupReloadable = async () => { await server.kill(); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index ca735dd3e..2a5c168f6 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -5,9 +5,10 @@ import { PostgresDatabaseService } from "@/database/postgres/service.js"; import type { DatabaseService, NamespaceInfo } from "@/database/service.js"; import { SqliteDatabaseService } from "@/database/sqlite/service.js"; import { getHistoricalStore } from "@/indexing-store/historical.js"; +import { getMetadataStore } from "@/indexing-store/metadata.js"; import { getReadonlyStore } from "@/indexing-store/readonly.js"; import { getRealtimeStore } from "@/indexing-store/realtime.js"; -import type { IndexingStore } from "@/indexing-store/store.js"; +import type { IndexingStore, Status } from "@/indexing-store/store.js"; import { createIndexingService } from "@/indexing/index.js"; import { createServer } from "@/server/service.js"; import { PostgresSyncStore } from "@/sync-store/postgres/store.js"; @@ -71,6 +72,15 @@ export async function run({ let namespaceInfo: NamespaceInfo; let initialCheckpoint: Checkpoint; + const status: Status = {}; + for (const network of networks) { + status[network.name] = { + ready: false, + blockNumber: null, + blockTimestamp: null, + }; + } + if (databaseConfig.kind === "sqlite") { const { directory } = databaseConfig; database = new SqliteDatabaseService({ common, directory }); @@ -94,6 +104,13 @@ export async function run({ syncStore = new PostgresSyncStore({ db: database.syncDb, common }); } + const metadataStore = getMetadataStore({ + encoding: database.kind, + namespaceInfo, + db: database.indexingDb, + }); + await metadataStore.setStatus(status); + const readonlyStore = getReadonlyStore({ encoding: database.kind, schema, @@ -102,7 +119,12 @@ export async function run({ common, }); - const server = await createServer({ common, graphqlSchema, readonlyStore }); + const server = await createServer({ + common, + graphqlSchema, + readonlyStore, + metadataStore, + }); // This can be a long-running operation, so it's best to do it after // starting the server so the app can become responsive more quickly. @@ -161,6 +183,25 @@ export async function run({ event.toCheckpoint, ); if (result.status === "error") onReloadableError(result.error); + + // set status to most recently processed realtime block or end block + // for each chain. + const checkpointStatus = syncService.getRealtimeStatus(); + for (const network of networks) { + status[network.name] = { + ready: true, + blockNumber: + checkpointStatus[network.name]?.blockNumber ?? + status[network.name]?.blockNumber ?? + null, + blockTimestamp: + checkpointStatus[network.name]?.blockTimestamp ?? + status[network.name]?.blockTimestamp ?? + null, + }; + } + + await metadataStore.setStatus(status); } break; @@ -268,12 +309,6 @@ export async function run({ await database.createIndexes({ schema }); - server.setHealthy(); - common.logger.info({ - service: "server", - msg: "Started responding as healthy", - }); - indexingStore = { ...readonlyStore, ...getRealtimeStore({ @@ -288,6 +323,24 @@ export async function run({ indexingService.updateIndexingStore({ indexingStore, schema }); syncService.startRealtime(); + + // set status to ready and set blocks to most recently processed + // or end block + const checkpointStatus = syncService.getRealtimeStatus(); + for (const network of networks) { + status[network.name] = { + ready: true, + blockNumber: checkpointStatus[network.name]?.blockNumber ?? null, + blockTimestamp: checkpointStatus[network.name]?.blockTimestamp ?? null, + }; + } + + await metadataStore.setStatus(status); + + common.logger.info({ + service: "server", + msg: "Started responding as healthy", + }); }; const startPromise = start(); diff --git a/packages/core/src/database/postgres/migrations.ts b/packages/core/src/database/postgres/migrations.ts index 85898234a..2f0c90f8d 100644 --- a/packages/core/src/database/postgres/migrations.ts +++ b/packages/core/src/database/postgres/migrations.ts @@ -1,3 +1,4 @@ +import type { Status } from "@/indexing-store/store.js"; import type { Kysely } from "kysely"; import type { Migration, MigrationProvider } from "kysely"; @@ -29,6 +30,9 @@ class StaticMigrationProvider implements MigrationProvider { export const migrationProvider = new StaticMigrationProvider(); export type InternalTables = { + _metadata: { + status: Status; + }; namespace_lock: { namespace: string; is_locked: number; diff --git a/packages/core/src/database/postgres/service.test.ts b/packages/core/src/database/postgres/service.test.ts index 9368298a2..e32862283 100644 --- a/packages/core/src/database/postgres/service.test.ts +++ b/packages/core/src/database/postgres/service.test.ts @@ -79,6 +79,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { ]); expect(await getTableNames(database.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", ]); @@ -113,6 +114,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", ]); @@ -127,6 +129,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "Dog", "Apple", ]); @@ -158,12 +161,14 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", ]); expect(await getViewNames(databaseTwo.db, "publish")).toStrictEqual([ "Pet", "Person", + "_metadata", ]); await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); @@ -177,12 +182,14 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "Dog", "Apple", ]); expect(await getViewNames(databaseTwo.db, "publish")).toStrictEqual([ "Dog", "Apple", + "_metadata", ]); await databaseTwo.kill(); @@ -215,6 +222,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { ); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", "not_a_ponder_table", @@ -224,6 +232,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "not_a_ponder_table", "AnotherTable", "Dog", @@ -562,6 +571,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public2", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ + "_metadata", "Dog", "Apple", ]); @@ -606,6 +616,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public2", "abc", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ + "_metadata", "Dog", "Apple", ]); @@ -626,6 +637,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", ]); @@ -635,6 +647,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getViewNames(database.db, "publish")).toStrictEqual([ "Pet", "Person", + "_metadata", ]); await database.kill(); @@ -652,6 +665,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", ]); @@ -666,6 +680,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getTableNames(database.db, "publish")).toStrictEqual(["Pet"]); expect(await getViewNames(database.db, "publish")).toStrictEqual([ "Person", + "_metadata", ]); await database.kill(); @@ -683,6 +698,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", ]); @@ -702,6 +718,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getViewNames(database.db, "nice_looks-great")).toStrictEqual([ "Pet", "Person", + "_metadata", ]); await database.kill(); diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index cdf14a09e..4def889ac 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -274,6 +274,15 @@ export class PostgresDatabaseService implements BaseDatabaseService { msg: `Acquired lock on new schema '${this.userNamespace}'`, }); + // create metadata table + await tx.schema + .withSchema(this.userNamespace) + .createTable("_metadata") + .addColumn("key", "text", (col) => col.primaryKey()) + .addColumn("value", "jsonb", (col) => col.notNull()) + .ifNotExists() + .execute(); + await createTables(); return { status: "success", checkpoint: zeroCheckpoint } as const; @@ -427,6 +436,12 @@ export class PostgresDatabaseService implements BaseDatabaseService { msg: `Acquired lock on schema '${this.userNamespace}' previously used by build '${previousBuildId}'`, }); + // clear metadata table + await tx + .withSchema(this.userNamespace) + .deleteFrom("_metadata") + .execute(); + for (const tableName of Object.keys(previousSchema.tables)) { const tableId = hash([ this.userNamespace, @@ -568,7 +583,9 @@ export class PostgresDatabaseService implements BaseDatabaseService { // Create the publish schema if it doesn't exist. await tx.schema.createSchema(publishSchema).ifNotExists().execute(); - for (const tableName of Object.keys(getTables(this.schema))) { + for (const tableName of Object.keys(getTables(this.schema)).concat( + "_metadata", + )) { // Check if there is an existing relation with the name we're about to publish. const result = await tx.executeQuery<{ table_type: string; diff --git a/packages/core/src/database/sqlite/migrations.ts b/packages/core/src/database/sqlite/migrations.ts index 0899726e7..e09f3a742 100644 --- a/packages/core/src/database/sqlite/migrations.ts +++ b/packages/core/src/database/sqlite/migrations.ts @@ -1,3 +1,4 @@ +import type { Status } from "@/indexing-store/store.js"; import type { Kysely } from "kysely"; import type { Migration, MigrationProvider } from "kysely"; @@ -29,6 +30,9 @@ class StaticMigrationProvider implements MigrationProvider { export const migrationProvider = new StaticMigrationProvider(); export type InternalTables = { + _metadata: { + status: Status; + }; namespace_lock: { namespace: string; is_locked: number; diff --git a/packages/core/src/database/sqlite/service.test.ts b/packages/core/src/database/sqlite/service.test.ts index 9251bec30..25390a190 100644 --- a/packages/core/src/database/sqlite/service.test.ts +++ b/packages/core/src/database/sqlite/service.test.ts @@ -78,6 +78,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { ]); expect(await getTableNames(database.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", ]); @@ -110,6 +111,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", ]); @@ -124,6 +126,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "Dog", "Apple", ]); @@ -156,6 +159,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { ); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "Pet", "Person", "not_a_ponder_table", @@ -165,6 +169,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ + "_metadata", "not_a_ponder_table", "AnotherTable", "Dog", @@ -489,6 +494,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public2", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ + "_metadata", "Dog", "Apple", ]); @@ -532,6 +538,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public2", "abc", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ + "_metadata", "Dog", "Apple", ]); diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index 058e0945a..6d8ba930a 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -245,6 +245,15 @@ export class SqliteDatabaseService implements BaseDatabaseService { msg: `Acquired lock on database file '${this.userNamespace}.db'`, }); + // create metadata table + await tx.schema + .withSchema(this.userNamespace) + .createTable("_metadata") + .addColumn("key", "text", (col) => col.primaryKey()) + .addColumn("value", "jsonb", (col) => col.notNull()) + .ifNotExists() + .execute(); + await createTables(); return { status: "success", checkpoint: zeroCheckpoint } as const; @@ -398,6 +407,12 @@ export class SqliteDatabaseService implements BaseDatabaseService { msg: `Acquired lock on schema '${this.userNamespace}' previously used by build '${previousBuildId}'`, }); + // clear metadata table + await tx + .withSchema(this.userNamespace) + .deleteFrom("_metadata") + .execute(); + for (const tableName of Object.keys(previousSchema.tables)) { const tableId = hash([ this.userNamespace, diff --git a/packages/core/src/indexing-store/metadata.test.ts b/packages/core/src/indexing-store/metadata.test.ts new file mode 100644 index 000000000..455300a2e --- /dev/null +++ b/packages/core/src/indexing-store/metadata.test.ts @@ -0,0 +1,59 @@ +import { + setupCommon, + setupDatabaseServices, + setupIsolatedDatabase, +} from "@/_test/setup.js"; +import { createSchema } from "@/schema/schema.js"; +import { beforeEach, expect, test } from "vitest"; +import { getMetadataStore } from "./metadata.js"; + +beforeEach(setupCommon); +beforeEach(setupIsolatedDatabase); + +const schema = createSchema(() => ({})); + +test("getMetadata() empty", async (context) => { + const { database, namespaceInfo, cleanup } = await setupDatabaseServices( + context, + { + schema, + }, + ); + const metadataStore = getMetadataStore({ + encoding: database.kind, + namespaceInfo, + db: database.indexingDb, + }); + + const status = await metadataStore.getStatus(); + + expect(status).toBeUndefined(); + + await cleanup(); +}); + +test("setMetadata()", async (context) => { + const { database, namespaceInfo, cleanup } = await setupDatabaseServices( + context, + { + schema, + }, + ); + const metadataStore = getMetadataStore({ + encoding: database.kind, + namespaceInfo, + db: database.indexingDb, + }); + + await metadataStore.setStatus({ + mainnet: { blockNumber: 10, blockTimestamp: 10, ready: false }, + }); + + const status = await metadataStore.getStatus(); + + expect(status).toStrictEqual({ + mainnet: { blockNumber: 10, blockTimestamp: 10, ready: false }, + }); + + await cleanup(); +}); diff --git a/packages/core/src/indexing-store/metadata.ts b/packages/core/src/indexing-store/metadata.ts new file mode 100644 index 000000000..46dd19a8e --- /dev/null +++ b/packages/core/src/indexing-store/metadata.ts @@ -0,0 +1,47 @@ +import type { HeadlessKysely } from "@/database/kysely.js"; +import type { NamespaceInfo } from "@/database/service.js"; +import type { MetadataStore, Status } from "./store.js"; + +export const getMetadataStore = ({ + encoding, + namespaceInfo, + db, +}: { + encoding: "sqlite" | "postgres"; + namespaceInfo: NamespaceInfo; + db: HeadlessKysely; +}): MetadataStore => ({ + getStatus: async () => { + return db.wrap({ method: "_metadata.getLatest()" }, async () => { + const metadata = await db + .withSchema(namespaceInfo.userNamespace) + .selectFrom("_metadata") + .select("value") + .where("key", "=", "status") + .executeTakeFirst(); + + if (metadata === undefined) return undefined; + + return encoding === "sqlite" + ? (JSON.parse(metadata.value) as Status) + : (metadata.value as Status); + }); + }, + setStatus: (status: Status) => { + return db.wrap({ method: "_metadata.setLatest()" }, async () => { + await db + .withSchema(namespaceInfo.userNamespace) + .insertInto("_metadata") + .values({ + key: "status", + value: encoding === "sqlite" ? JSON.stringify(status) : status, + }) + .onConflict((oc) => + oc.column("key").doUpdateSet({ + value: encoding === "sqlite" ? JSON.stringify(status) : status, + }), + ) + .execute(); + }); + }, +}); diff --git a/packages/core/src/indexing-store/store.ts b/packages/core/src/indexing-store/store.ts index 2af76f254..b641323d1 100644 --- a/packages/core/src/indexing-store/store.ts +++ b/packages/core/src/indexing-store/store.ts @@ -99,6 +99,19 @@ export type HistoricalStore = ReadonlyStore & flush: (arg: { isFullFlush: boolean }) => Promise; }; +export type Status = { + [networkName: string]: { + blockNumber: number | null; + blockTimestamp: number | null; + ready: boolean; + }; +}; + +export type MetadataStore = { + setStatus: (status: Status) => Promise; + getStatus: () => Promise; +}; + export type IndexingStore< env extends "historical" | "realtime" = "historical" | "realtime", > = ReadonlyStore & WriteStore; diff --git a/packages/core/src/server/service.test.ts b/packages/core/src/server/service.test.ts index 1bfeb6c7f..feec0cd91 100644 --- a/packages/core/src/server/service.test.ts +++ b/packages/core/src/server/service.test.ts @@ -3,7 +3,11 @@ import { setupDatabaseServices, setupIsolatedDatabase, } from "@/_test/setup.js"; -import type { HistoricalStore, ReadonlyStore } from "@/indexing-store/store.js"; +import type { + HistoricalStore, + MetadataStore, + ReadonlyStore, +} from "@/indexing-store/store.js"; import { createSchema } from "@/schema/schema.js"; import { encodeCheckpoint, zeroCheckpoint } from "@/utils/checkpoint.js"; import type { GraphQLSchema } from "graphql"; @@ -14,17 +18,26 @@ import { createServer } from "./service.js"; beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); +const getMockMetadataStore = (ready: boolean) => + ({ + getStatus() { + return Promise.resolve({ mainnet: { ready } }); + }, + }) as unknown as MetadataStore; + test("port", async (context) => { const server1 = await createServer({ graphqlSchema: {} as GraphQLSchema, common: context.common, readonlyStore: {} as ReadonlyStore, + metadataStore: {} as MetadataStore, }); const server2 = await createServer({ graphqlSchema: {} as GraphQLSchema, common: context.common, readonlyStore: {} as ReadonlyStore, + metadataStore: {} as MetadataStore, }); expect(server2.port).toBeGreaterThanOrEqual(server1.port + 1); @@ -41,6 +54,7 @@ test("not healthy", async (context) => { options: { ...context.common.options, maxHealthcheckDuration: 5 }, }, readonlyStore: {} as ReadonlyStore, + metadataStore: getMockMetadataStore(false), }); const response = await server.hono.request("/health"); @@ -58,6 +72,7 @@ test("healthy", async (context) => { options: { ...context.common.options, maxHealthcheckDuration: 0 }, }, readonlyStore: {} as ReadonlyStore, + metadataStore: getMockMetadataStore(true), }); const response = await server.hono.request("/health"); @@ -75,6 +90,7 @@ test("healthy PUT", async (context) => { options: { ...context.common.options, maxHealthcheckDuration: 0 }, }, readonlyStore: {} as ReadonlyStore, + metadataStore: {} as MetadataStore, }); const response = await server.hono.request("/health", { method: "PUT" }); @@ -89,6 +105,7 @@ test("metrics", async (context) => { graphqlSchema: {} as GraphQLSchema, common: context.common, readonlyStore: {} as ReadonlyStore, + metadataStore: {} as MetadataStore, }); const response = await server.hono.request("/metrics"); @@ -103,6 +120,7 @@ test("metrics error", async (context) => { graphqlSchema: {} as GraphQLSchema, common: context.common, readonlyStore: {} as ReadonlyStore, + metadataStore: {} as MetadataStore, }); const metricsSpy = vi.spyOn(context.common.metrics, "getMetrics"); @@ -120,6 +138,7 @@ test("metrics PUT", async (context) => { graphqlSchema: {} as GraphQLSchema, common: context.common, readonlyStore: {} as ReadonlyStore, + metadataStore: {} as MetadataStore, }); const response = await server.hono.request("/metrics", { method: "PUT" }); @@ -171,8 +190,8 @@ test("graphql", async (context) => { graphqlSchema: graphqlSchema, common: context.common, readonlyStore: readonlyStore, + metadataStore: getMockMetadataStore(true), }); - server.setHealthy(); const response = await server.hono.request("/graphql", { method: "POST", @@ -240,8 +259,8 @@ test("graphql extra filter", async (context) => { graphqlSchema: graphqlSchema, common: context.common, readonlyStore: readonlyStore, + metadataStore: getMockMetadataStore(true), }); - server.setHealthy(); const response = await server.hono.request("/graphql", { method: "POST", @@ -292,8 +311,8 @@ test("graphql token limit error", async (context) => { options: { ...context.common.options, graphqlMaxOperationTokens: 3 }, }, readonlyStore: readonlyStore, + metadataStore: getMockMetadataStore(true), }); - server.setHealthy(); const response = await server.hono.request("/graphql", { method: "POST", @@ -348,8 +367,8 @@ test("graphql depth limit error", async (context) => { options: { ...context.common.options, graphqlMaxOperationDepth: 5 }, }, readonlyStore: readonlyStore, + metadataStore: getMockMetadataStore(true), }); - server.setHealthy(); const response = await server.hono.request("/graphql", { method: "POST", @@ -404,8 +423,8 @@ test("graphql max aliases error", async (context) => { options: { ...context.common.options, graphqlMaxOperationAliases: 2 }, }, readonlyStore: readonlyStore, + metadataStore: getMockMetadataStore(true), }); - server.setHealthy(); const response = await server.hono.request("/graphql", { method: "POST", @@ -457,8 +476,8 @@ test("graphql interactive", async (context) => { graphqlSchema: {} as GraphQLSchema, common: context.common, readonlyStore: {} as ReadonlyStore, + metadataStore: {} as MetadataStore, }); - server.setHealthy(); const response = await server.hono.request("/graphql"); @@ -472,6 +491,7 @@ test("missing route", async (context) => { graphqlSchema: {} as GraphQLSchema, common: context.common, readonlyStore: {} as ReadonlyStore, + metadataStore: {} as MetadataStore, }); const response = await server.hono.request("/kevin"); @@ -488,6 +508,7 @@ test.skip("kill", async (context) => { graphqlSchema: {} as GraphQLSchema, common: context.common, readonlyStore: {} as ReadonlyStore, + metadataStore: {} as MetadataStore, }); await server.kill(); diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index be2497220..d6444e208 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -1,6 +1,6 @@ import http from "node:http"; import type { Common } from "@/common/common.js"; -import type { ReadonlyStore } from "@/indexing-store/store.js"; +import type { MetadataStore, ReadonlyStore } from "@/indexing-store/store.js"; import { graphiQLHtml } from "@/ui/graphiql.html.js"; import { startClock } from "@/utils/timer.js"; import { maxAliasesPlugin } from "@escape.tech/graphql-armor-max-aliases"; @@ -21,17 +21,18 @@ import { type Server = { hono: Hono<{ Variables: { store: ReadonlyStore; getLoader: GetLoader } }>; port: number; - setHealthy: () => void; kill: () => Promise; }; export async function createServer({ graphqlSchema, readonlyStore, + metadataStore, common, }: { graphqlSchema: GraphQLSchema; readonlyStore: ReadonlyStore; + metadataStore: MetadataStore; common: Common; }): Promise { const hono = new Hono<{ @@ -39,7 +40,6 @@ export async function createServer({ }>(); let port = common.options.port; - let isHealthy = false; const startTime = Date.now(); const metricsMiddleware = createMiddleware(async (c, next) => { @@ -121,7 +121,12 @@ export async function createServer({ } }) .get("/health", async (c) => { - if (isHealthy) { + const status = await metadataStore.getStatus(); + + if ( + status !== undefined && + Object.values(status).every(({ ready }) => ready === true) + ) { return c.text("", 200); } @@ -141,8 +146,12 @@ export async function createServer({ // Renders GraphiQL .get("/graphql", (c) => c.html(prodGraphiql)) // Serves GraphQL POST requests following healthcheck rules - .post("/graphql", (c) => { - if (isHealthy === false) { + .post("/graphql", async (c) => { + const status = await metadataStore.getStatus(); + if ( + status === undefined || + Object.values(status).some(({ ready }) => ready === false) + ) { return c.json( { errors: [new GraphQLError("Historical indexing is not complete")] }, 503, @@ -154,7 +163,12 @@ export async function createServer({ // Renders GraphiQL .get("/", (c) => c.html(rootGraphiql)) // Serves GraphQL POST requests regardless of health status, e.g. "dev UI" - .post("/", (c) => rootYoga.handle(c.req.raw)); + .post("/", (c) => rootYoga.handle(c.req.raw)) + .get("/status", async (c) => { + const status = await metadataStore.getStatus(); + + return c.json(status); + }); const createServerWithNextAvailablePort: typeof http.createServer = ( ...args: any @@ -220,9 +234,7 @@ export async function createServer({ return { hono, port, - setHealthy: () => { - isHealthy = true; - }, + kill: () => terminator.terminate(), }; } diff --git a/packages/core/src/sync-realtime/index.ts b/packages/core/src/sync-realtime/index.ts index 77d8544a2..69ddb4f34 100644 --- a/packages/core/src/sync-realtime/index.ts +++ b/packages/core/src/sync-realtime/index.ts @@ -1,10 +1,11 @@ import { type Extend, extend } from "@/utils/extend.js"; -import { create, kill, start } from "./service.js"; +import { create, getMostRecentBlock, kill, start } from "./service.js"; import type { RealtimeSyncEvent, Service } from "./service.js"; const methods = { start, kill, + getMostRecentBlock, }; export const createRealtimeSyncService = extend(create, methods); diff --git a/packages/core/src/sync-realtime/service.ts b/packages/core/src/sync-realtime/service.ts index 2af8bede7..cc312d5d5 100644 --- a/packages/core/src/sync-realtime/service.ts +++ b/packages/core/src/sync-realtime/service.ts @@ -657,6 +657,26 @@ export const handleReorg = async ( throw new Error(msg); }; +/** + * Find the most recent block that is less than or equal to + * the provided checkpoint. + */ +export const getMostRecentBlock = ( + service: Service, + checkpoint: Checkpoint, +): LightBlock | undefined => { + const localBlock = service.localChain.findLast( + (block) => block.timestamp <= checkpoint.blockTimestamp, + ); + + if (localBlock !== undefined) return localBlock; + + if (service.finalizedBlock.timestamp < checkpoint.blockTimestamp) + return service.finalizedBlock; + + return undefined; +}; + const getMatchedLogs = async ( service: Service, { diff --git a/packages/core/src/sync/index.ts b/packages/core/src/sync/index.ts index 13927ea77..5de371440 100644 --- a/packages/core/src/sync/index.ts +++ b/packages/core/src/sync/index.ts @@ -22,6 +22,7 @@ import { create, getCachedTransport, getHistoricalCheckpoint, + getRealtimeStatus, kill, startHistorical, startRealtime, @@ -30,6 +31,7 @@ import { const methods = { startHistorical, getHistoricalCheckpoint, + getRealtimeStatus, startRealtime, getCachedTransport, kill, diff --git a/packages/core/src/sync/service.ts b/packages/core/src/sync/service.ts index 58cfc1f26..699b6d1e6 100644 --- a/packages/core/src/sync/service.ts +++ b/packages/core/src/sync/service.ts @@ -566,6 +566,41 @@ export const kill = async (syncService: Service) => { await Promise.all(killPromise); }; +/** Return the number and timestamp of the most recently processed blocks. */ +export const getRealtimeStatus = (syncService: Service) => { + const status: { + [networkName: string]: + | { blockNumber: number; blockTimestamp: number } + | undefined; + } = {}; + + for (const networkService of syncService.networkServices) { + if (networkService.realtime === undefined) { + status[networkService.network.name] = { + blockNumber: Number(networkService.endCheckpoint!.blockNumber), + blockTimestamp: networkService.endCheckpoint!.blockTimestamp, + }; + } else { + const mostRecentBlock = + networkService.realtime.realtimeSync.getMostRecentBlock( + syncService.checkpoint, + ); + + if (mostRecentBlock === undefined) { + status[networkService.network.name] = undefined; + } else { + status[networkService.network.name] = { + ...zeroCheckpoint, + blockTimestamp: mostRecentBlock.timestamp, + blockNumber: mostRecentBlock.number, + }; + } + } + } + + return status; +}; + export const getCachedTransport = (syncService: Service, network: Network) => { const { requestQueue } = syncService.networkServices.find( (ns) => ns.network.chainId === network.chainId, From 2d0e3c5135ed87f5616b83c9447b48e2baafa1ff Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 10 Jul 2024 13:25:43 -0400 Subject: [PATCH 065/122] graphql metadata entity --- .../server/graphql/buildGraphqlSchema.test.ts | 148 ++++++++++++------ .../src/server/graphql/buildGraphqlSchema.ts | 17 +- packages/core/src/server/graphql/entity.ts | 2 +- packages/core/src/server/graphql/metadata.ts | 7 + packages/core/src/server/graphql/plural.ts | 2 +- packages/core/src/server/graphql/singular.ts | 5 +- packages/core/src/server/service.ts | 2 +- 7 files changed, 132 insertions(+), 51 deletions(-) create mode 100644 packages/core/src/server/graphql/metadata.ts diff --git a/packages/core/src/server/graphql/buildGraphqlSchema.test.ts b/packages/core/src/server/graphql/buildGraphqlSchema.test.ts index 1d27bbd20..f1a0298d8 100644 --- a/packages/core/src/server/graphql/buildGraphqlSchema.test.ts +++ b/packages/core/src/server/graphql/buildGraphqlSchema.test.ts @@ -3,6 +3,7 @@ import { setupDatabaseServices, setupIsolatedDatabase, } from "@/_test/setup.js"; +import { getMetadataStore } from "@/indexing-store/metadata.js"; import type { IndexingStore } from "@/indexing-store/store.js"; import { createSchema } from "@/schema/schema.js"; import { encodeCheckpoint, zeroCheckpoint } from "@/utils/checkpoint.js"; @@ -68,7 +69,7 @@ test("scalar", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -136,7 +137,7 @@ test("scalar list", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -204,7 +205,7 @@ test("scalar optional", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -272,7 +273,7 @@ test("scalar optional list", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -325,7 +326,7 @@ test("json", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -374,7 +375,7 @@ test("enum", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -423,7 +424,7 @@ test("enum optional", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -481,7 +482,7 @@ test("enum list", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -539,7 +540,7 @@ test("enum optional list", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -598,7 +599,7 @@ test("one", async (context) => { schema: graphqlSchema, document, contextValue: { - store: indexingStore, + readonlyStore: indexingStore, getLoader: buildLoaderCache({ store: indexingStore }), }, }); @@ -668,7 +669,7 @@ test("many", async (context) => { schema: graphqlSchema, document, contextValue: { - store: indexingStore, + readonlyStore: indexingStore, getLoader: buildLoaderCache({ store: indexingStore }), }, }); @@ -749,7 +750,7 @@ test("many w/ filter", async (context) => { schema: graphqlSchema, document, contextValue: { - store: indexingStore, + readonlyStore: indexingStore, getLoader: buildLoaderCache({ store: indexingStore }), }, }); @@ -799,7 +800,7 @@ test("bigint id", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -841,7 +842,7 @@ test("hex id", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -893,7 +894,7 @@ test("filter string eq", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -955,7 +956,7 @@ test("filter string in", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1029,7 +1030,7 @@ test("filter string contains", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1103,7 +1104,7 @@ test("filter string starts with", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1177,7 +1178,7 @@ test("filter string not ends with", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1239,7 +1240,7 @@ test("filter int eq", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1313,7 +1314,7 @@ test("filter int gt", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1375,7 +1376,7 @@ test("filter int lte", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1437,7 +1438,7 @@ test("filter int in", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1499,7 +1500,7 @@ test("filter float eq", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1573,7 +1574,7 @@ test("filter float gt", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1635,7 +1636,7 @@ test("filter float lte", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1697,7 +1698,7 @@ test("filter float in", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1759,7 +1760,7 @@ test("filter bigint eq", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1833,7 +1834,7 @@ test("filter bigint gt", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1895,7 +1896,7 @@ test("filter bigint lte", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -1957,7 +1958,7 @@ test("filter bigint in", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2019,7 +2020,7 @@ test("filer hex eq", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2093,7 +2094,7 @@ test("filter hex gt", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2167,7 +2168,7 @@ test("filter string list eq", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2241,7 +2242,7 @@ test("filter string list has", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2301,7 +2302,7 @@ test("filter enum eq", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2356,7 +2357,7 @@ test("filter enum in", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2419,7 +2420,7 @@ test("filter ref eq", async (context) => { schema: graphqlSchema, document, contextValue: { - store: indexingStore, + readonlyStore: indexingStore, getLoader: buildLoaderCache({ store: indexingStore }), }, }); @@ -2486,7 +2487,7 @@ test("filter ref in", async (context) => { schema: graphqlSchema, document, contextValue: { - store: indexingStore, + readonlyStore: indexingStore, getLoader: buildLoaderCache({ store: indexingStore }), }, }); @@ -2568,7 +2569,7 @@ test("order int asc", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2652,7 +2653,7 @@ test("order bigint asc", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2736,7 +2737,7 @@ test("order bigint desc", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); expect(result.data).toMatchObject({ @@ -2800,7 +2801,7 @@ test("limit default", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); // @ts-ignore @@ -2851,7 +2852,7 @@ test("limit", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); // @ts-ignore @@ -2902,7 +2903,7 @@ test("limit error", async (context) => { const result = await execute({ schema: graphqlSchema, document, - contextValue: { store: indexingStore }, + contextValue: { readonlyStore: indexingStore }, }); // @ts-ignore @@ -3036,3 +3037,60 @@ test("filter type has correct suffixes and types", () => { relatedTableBigIntId_lte: "BigInt", }); }); + +test("metadata", async (context) => { + const schema = createSchema(() => ({})); + + const { indexingStore, cleanup, database, namespaceInfo } = + await setupDatabaseServices(context, { + schema, + }); + + const metadataStore = getMetadataStore({ + encoding: database.kind, + db: database.readonlyDb, + namespaceInfo, + }); + + await metadataStore.setStatus({ + mainnet: { + ready: true, + blockNumber: 10, + blockTimestamp: 20, + }, + }); + + const graphqlSchema = buildGraphqlSchema(schema); + + const document = parse(` + query { + _metadata { + status + } + } + `); + + const result = await execute({ + schema: graphqlSchema, + document, + contextValue: { readonlyStore: indexingStore, metadataStore }, + }); + + expect(result.data).toMatchObject({ + _metadata: { + status: { + mainnet: { + ready: true, + blockNumber: 10, + blockTimestamp: 20, + }, + }, + }, + }); + + console.log(JSON.stringify(result, null, 2)); + + // console.log(result.data._metadata); + + await cleanup(); +}); diff --git a/packages/core/src/server/graphql/buildGraphqlSchema.ts b/packages/core/src/server/graphql/buildGraphqlSchema.ts index 36164867e..8bdee6443 100644 --- a/packages/core/src/server/graphql/buildGraphqlSchema.ts +++ b/packages/core/src/server/graphql/buildGraphqlSchema.ts @@ -1,4 +1,4 @@ -import type { IndexingStore } from "@/indexing-store/store.js"; +import type { MetadataStore, ReadonlyStore } from "@/indexing-store/store.js"; import type { Schema } from "@/schema/common.js"; import { getTables } from "@/schema/utils.js"; import { @@ -10,12 +10,17 @@ import type { GetLoader } from "./buildLoaderCache.js"; import { buildEntityTypes } from "./entity.js"; import { buildEnumTypes } from "./enum.js"; import { buildEntityFilterTypes } from "./filter.js"; +import { metadataEntity } from "./metadata.js"; import { buildPluralField } from "./plural.js"; import { buildSingularField } from "./singular.js"; // TODO(kyle) stricter type export type Parent = Record; -export type Context = { store: IndexingStore; getLoader: GetLoader }; +export type Context = { + getLoader: GetLoader; + readonlyStore: ReadonlyStore; + metadataStore: MetadataStore; +}; export const buildGraphqlSchema = (schema: Schema): GraphQLSchema => { const queryFields: Record> = {}; @@ -49,6 +54,14 @@ export const buildGraphqlSchema = (schema: Schema): GraphQLSchema => { }); } + queryFields._metadata = { + type: metadataEntity, + resolve: async (_source, _args, context) => { + const status = await context.metadataStore.getStatus(); + return { status }; + }, + }; + return new GraphQLSchema({ query: new GraphQLObjectType({ name: "Query", diff --git a/packages/core/src/server/graphql/entity.ts b/packages/core/src/server/graphql/entity.ts index 9c57eab74..351779059 100644 --- a/packages/core/src/server/graphql/entity.ts +++ b/packages/core/src/server/graphql/entity.ts @@ -107,7 +107,7 @@ export const buildEntityTypes = ({ // Query for the IDs of the matching records. // TODO: Update query to only fetch IDs, not entire records. - const result = await context.store.findMany({ + const result = await context.readonlyStore.findMany({ tableName: column[" referenceTable"], where: whereObject, orderBy: orderByObject, diff --git a/packages/core/src/server/graphql/metadata.ts b/packages/core/src/server/graphql/metadata.ts new file mode 100644 index 000000000..933f9705c --- /dev/null +++ b/packages/core/src/server/graphql/metadata.ts @@ -0,0 +1,7 @@ +import { GraphQLObjectType } from "graphql"; +import { GraphQLJSON } from "graphql-type-json"; + +export const metadataEntity = new GraphQLObjectType({ + name: "_metadata", + fields: { status: { type: GraphQLJSON } }, +}); diff --git a/packages/core/src/server/graphql/plural.ts b/packages/core/src/server/graphql/plural.ts index 323857331..a10161759 100644 --- a/packages/core/src/server/graphql/plural.ts +++ b/packages/core/src/server/graphql/plural.ts @@ -39,7 +39,7 @@ export const buildPluralField = ({ ? { [orderBy]: orderDirection || "asc" } : undefined; - return await context.store.findMany({ + return await context.readonlyStore.findMany({ tableName, where: whereObject, orderBy: orderByObject, diff --git a/packages/core/src/server/graphql/singular.ts b/packages/core/src/server/graphql/singular.ts index 7cf1377f8..1f9076846 100644 --- a/packages/core/src/server/graphql/singular.ts +++ b/packages/core/src/server/graphql/singular.ts @@ -27,7 +27,10 @@ export const buildSingularField = ({ if (id === undefined) return null; - const entityInstance = await context.store.findUnique({ tableName, id }); + const entityInstance = await context.readonlyStore.findUnique({ + tableName, + id, + }); return entityInstance; }; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index d6444e208..feb23e7f0 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -83,7 +83,7 @@ export async function createServer({ schema: graphqlSchema, context: () => { const getLoader = buildLoaderCache({ store: readonlyStore }); - return { store: readonlyStore, getLoader }; + return { getLoader, readonlyStore, metadataStore }; }, graphqlEndpoint: path, maskedErrors: process.env.NODE_ENV === "production", From 9809d14f14744b92564f00a0425e5e5fc6841f67 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 10 Jul 2024 13:31:56 -0400 Subject: [PATCH 066/122] nits --- packages/core/src/database/postgres/migrations.ts | 3 +-- packages/core/src/database/sqlite/migrations.ts | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/core/src/database/postgres/migrations.ts b/packages/core/src/database/postgres/migrations.ts index 2f0c90f8d..7539d0867 100644 --- a/packages/core/src/database/postgres/migrations.ts +++ b/packages/core/src/database/postgres/migrations.ts @@ -1,4 +1,3 @@ -import type { Status } from "@/indexing-store/store.js"; import type { Kysely } from "kysely"; import type { Migration, MigrationProvider } from "kysely"; @@ -31,7 +30,7 @@ export const migrationProvider = new StaticMigrationProvider(); export type InternalTables = { _metadata: { - status: Status; + key: object; }; namespace_lock: { namespace: string; diff --git a/packages/core/src/database/sqlite/migrations.ts b/packages/core/src/database/sqlite/migrations.ts index e09f3a742..db853d47e 100644 --- a/packages/core/src/database/sqlite/migrations.ts +++ b/packages/core/src/database/sqlite/migrations.ts @@ -1,4 +1,3 @@ -import type { Status } from "@/indexing-store/store.js"; import type { Kysely } from "kysely"; import type { Migration, MigrationProvider } from "kysely"; @@ -31,7 +30,7 @@ export const migrationProvider = new StaticMigrationProvider(); export type InternalTables = { _metadata: { - status: Status; + key: object; }; namespace_lock: { namespace: string; From 9346d82c1d777994612983c57eab4f78bbdbf692 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 10 Jul 2024 14:59:15 -0400 Subject: [PATCH 067/122] chore: changeset --- .changeset/silent-walls-give.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/silent-walls-give.md diff --git a/.changeset/silent-walls-give.md b/.changeset/silent-walls-give.md new file mode 100644 index 000000000..a5053d66a --- /dev/null +++ b/.changeset/silent-walls-give.md @@ -0,0 +1,5 @@ +--- +"@ponder/core": patch +--- + +Added per network progress to the "/status" endpoint on the server and the "_metadata" entity on the GraphQL schema. From 542e7aa93534c8d3d4fce23e84426728a79d1f86 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 10 Jul 2024 15:40:19 -0400 Subject: [PATCH 068/122] fix: kysely types --- packages/core/src/database/postgres/migrations.ts | 3 --- packages/core/src/database/postgres/service.ts | 1 + packages/core/src/database/sqlite/migrations.ts | 3 --- packages/core/src/database/sqlite/service.ts | 1 + 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/core/src/database/postgres/migrations.ts b/packages/core/src/database/postgres/migrations.ts index 7539d0867..85898234a 100644 --- a/packages/core/src/database/postgres/migrations.ts +++ b/packages/core/src/database/postgres/migrations.ts @@ -29,9 +29,6 @@ class StaticMigrationProvider implements MigrationProvider { export const migrationProvider = new StaticMigrationProvider(); export type InternalTables = { - _metadata: { - key: object; - }; namespace_lock: { namespace: string; is_locked: number; diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index 4def889ac..1c52cf57e 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -439,6 +439,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { // clear metadata table await tx .withSchema(this.userNamespace) + // @ts-expect-error Kysely doesn't have types for user schema .deleteFrom("_metadata") .execute(); diff --git a/packages/core/src/database/sqlite/migrations.ts b/packages/core/src/database/sqlite/migrations.ts index db853d47e..0899726e7 100644 --- a/packages/core/src/database/sqlite/migrations.ts +++ b/packages/core/src/database/sqlite/migrations.ts @@ -29,9 +29,6 @@ class StaticMigrationProvider implements MigrationProvider { export const migrationProvider = new StaticMigrationProvider(); export type InternalTables = { - _metadata: { - key: object; - }; namespace_lock: { namespace: string; is_locked: number; diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index 6d8ba930a..4c27e8b4b 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -410,6 +410,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { // clear metadata table await tx .withSchema(this.userNamespace) + // @ts-expect-error Kysely doesn't have types for user schema .deleteFrom("_metadata") .execute(); From 0ee458abef38ca172e914cb3c1966c1bf133ac11 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 10 Jul 2024 15:46:38 -0400 Subject: [PATCH 069/122] fix: ponder_metadata table name --- .../src/database/postgres/service.test.ts | 34 +++++++++---------- .../core/src/database/postgres/service.ts | 6 ++-- .../core/src/database/sqlite/service.test.ts | 14 ++++---- packages/core/src/database/sqlite/service.ts | 4 +-- packages/core/src/indexing-store/metadata.ts | 4 +-- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/packages/core/src/database/postgres/service.test.ts b/packages/core/src/database/postgres/service.test.ts index e32862283..597a3a138 100644 --- a/packages/core/src/database/postgres/service.test.ts +++ b/packages/core/src/database/postgres/service.test.ts @@ -79,7 +79,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { ]); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", ]); @@ -114,7 +114,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", ]); @@ -129,7 +129,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Dog", "Apple", ]); @@ -161,14 +161,14 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", ]); expect(await getViewNames(databaseTwo.db, "publish")).toStrictEqual([ "Pet", "Person", - "_metadata", + "ponder_metadata", ]); await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); @@ -182,14 +182,14 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Dog", "Apple", ]); expect(await getViewNames(databaseTwo.db, "publish")).toStrictEqual([ "Dog", "Apple", - "_metadata", + "ponder_metadata", ]); await databaseTwo.kill(); @@ -222,7 +222,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { ); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", "not_a_ponder_table", @@ -232,7 +232,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "not_a_ponder_table", "AnotherTable", "Dog", @@ -571,7 +571,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public2", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Dog", "Apple", ]); @@ -616,7 +616,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public2", "abc", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Dog", "Apple", ]); @@ -637,7 +637,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", ]); @@ -647,7 +647,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getViewNames(database.db, "publish")).toStrictEqual([ "Pet", "Person", - "_metadata", + "ponder_metadata", ]); await database.kill(); @@ -665,7 +665,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", ]); @@ -680,7 +680,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getTableNames(database.db, "publish")).toStrictEqual(["Pet"]); expect(await getViewNames(database.db, "publish")).toStrictEqual([ "Person", - "_metadata", + "ponder_metadata", ]); await database.kill(); @@ -698,7 +698,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", ]); @@ -718,7 +718,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getViewNames(database.db, "nice_looks-great")).toStrictEqual([ "Pet", "Person", - "_metadata", + "ponder_metadata", ]); await database.kill(); diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index 1c52cf57e..a9a5f766f 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -277,7 +277,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { // create metadata table await tx.schema .withSchema(this.userNamespace) - .createTable("_metadata") + .createTable("ponder_metadata") .addColumn("key", "text", (col) => col.primaryKey()) .addColumn("value", "jsonb", (col) => col.notNull()) .ifNotExists() @@ -440,7 +440,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { await tx .withSchema(this.userNamespace) // @ts-expect-error Kysely doesn't have types for user schema - .deleteFrom("_metadata") + .deleteFrom("ponder_metadata") .execute(); for (const tableName of Object.keys(previousSchema.tables)) { @@ -585,7 +585,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { await tx.schema.createSchema(publishSchema).ifNotExists().execute(); for (const tableName of Object.keys(getTables(this.schema)).concat( - "_metadata", + "ponder_metadata", )) { // Check if there is an existing relation with the name we're about to publish. const result = await tx.executeQuery<{ diff --git a/packages/core/src/database/sqlite/service.test.ts b/packages/core/src/database/sqlite/service.test.ts index 25390a190..0d9a65901 100644 --- a/packages/core/src/database/sqlite/service.test.ts +++ b/packages/core/src/database/sqlite/service.test.ts @@ -78,7 +78,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { ]); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", ]); @@ -111,7 +111,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", ]); @@ -126,7 +126,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Dog", "Apple", ]); @@ -159,7 +159,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { ); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Pet", "Person", "not_a_ponder_table", @@ -169,7 +169,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_metadata", + "ponder_metadata", "not_a_ponder_table", "AnotherTable", "Dog", @@ -494,7 +494,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public2", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Dog", "Apple", ]); @@ -538,7 +538,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public2", "abc", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "_metadata", + "ponder_metadata", "Dog", "Apple", ]); diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index 4c27e8b4b..e7ff92aa6 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -248,7 +248,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { // create metadata table await tx.schema .withSchema(this.userNamespace) - .createTable("_metadata") + .createTable("ponder_metadata") .addColumn("key", "text", (col) => col.primaryKey()) .addColumn("value", "jsonb", (col) => col.notNull()) .ifNotExists() @@ -411,7 +411,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { await tx .withSchema(this.userNamespace) // @ts-expect-error Kysely doesn't have types for user schema - .deleteFrom("_metadata") + .deleteFrom("ponder_metadata") .execute(); for (const tableName of Object.keys(previousSchema.tables)) { diff --git a/packages/core/src/indexing-store/metadata.ts b/packages/core/src/indexing-store/metadata.ts index 46dd19a8e..141d879df 100644 --- a/packages/core/src/indexing-store/metadata.ts +++ b/packages/core/src/indexing-store/metadata.ts @@ -15,7 +15,7 @@ export const getMetadataStore = ({ return db.wrap({ method: "_metadata.getLatest()" }, async () => { const metadata = await db .withSchema(namespaceInfo.userNamespace) - .selectFrom("_metadata") + .selectFrom("ponder_metadata") .select("value") .where("key", "=", "status") .executeTakeFirst(); @@ -31,7 +31,7 @@ export const getMetadataStore = ({ return db.wrap({ method: "_metadata.setLatest()" }, async () => { await db .withSchema(namespaceInfo.userNamespace) - .insertInto("_metadata") + .insertInto("ponder_metadata") .values({ key: "status", value: encoding === "sqlite" ? JSON.stringify(status) : status, From 95f18e401e65571dbb8acc8c66d8327d1169b30e Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 10 Jul 2024 16:22:03 -0400 Subject: [PATCH 070/122] fix: database ponder_metadata creation and initialization --- .../core/src/database/postgres/service.ts | 36 ++++++++++--------- packages/core/src/database/sqlite/service.ts | 36 ++++++++++--------- packages/core/src/indexing-store/metadata.ts | 6 ++-- packages/core/src/indexing-store/store.ts | 2 +- packages/core/src/server/service.ts | 4 +-- 5 files changed, 46 insertions(+), 38 deletions(-) diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index a9a5f766f..88bafe169 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -262,6 +262,26 @@ export class PostgresDatabaseService implements BaseDatabaseService { } }; + // Create ponder_metadata table if it doesn't exist + await tx.schema + .withSchema(this.userNamespace) + .createTable("ponder_metadata") + .addColumn("key", "text", (col) => col.primaryKey()) + .addColumn("value", "jsonb") + .ifNotExists() + .execute(); + + // Create or set status to null + await tx + .withSchema(this.userNamespace) + // @ts-expect-error Kysely doesn't have types for user schema + .insertInto("ponder_metadata") + // @ts-expect-error Kysely doesn't have types for user schema + .values({ key: "status", value: null }) + // @ts-expect-error Kysely doesn't have types for user schema + .onConflict((oc) => oc.column("key").doUpdateSet({ value: null })) + .execute(); + // If no lock row is found for this namespace, we can acquire the lock. if (previousLockRow === undefined) { await tx @@ -274,15 +294,6 @@ export class PostgresDatabaseService implements BaseDatabaseService { msg: `Acquired lock on new schema '${this.userNamespace}'`, }); - // create metadata table - await tx.schema - .withSchema(this.userNamespace) - .createTable("ponder_metadata") - .addColumn("key", "text", (col) => col.primaryKey()) - .addColumn("value", "jsonb", (col) => col.notNull()) - .ifNotExists() - .execute(); - await createTables(); return { status: "success", checkpoint: zeroCheckpoint } as const; @@ -436,13 +447,6 @@ export class PostgresDatabaseService implements BaseDatabaseService { msg: `Acquired lock on schema '${this.userNamespace}' previously used by build '${previousBuildId}'`, }); - // clear metadata table - await tx - .withSchema(this.userNamespace) - // @ts-expect-error Kysely doesn't have types for user schema - .deleteFrom("ponder_metadata") - .execute(); - for (const tableName of Object.keys(previousSchema.tables)) { const tableId = hash([ this.userNamespace, diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index e7ff92aa6..5b767d215 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -233,6 +233,26 @@ export class SqliteDatabaseService implements BaseDatabaseService { } }; + // Create ponder_metadata table if it doesn't exist + await tx.schema + .withSchema(this.userNamespace) + .createTable("ponder_metadata") + .addColumn("key", "text", (col) => col.primaryKey()) + .addColumn("value", "jsonb") + .ifNotExists() + .execute(); + + // Create or set status to null + await tx + .withSchema(this.userNamespace) + // @ts-expect-error Kysely doesn't have types for user schema + .insertInto("ponder_metadata") + // @ts-expect-error Kysely doesn't have types for user schema + .values({ key: "status", value: null }) + // @ts-expect-error Kysely doesn't have types for user schema + .onConflict((oc) => oc.column("key").doUpdateSet({ value: null })) + .execute(); + // If no lock row is found for this namespace, we can acquire the lock. if (previousLockRow === undefined) { await tx @@ -245,15 +265,6 @@ export class SqliteDatabaseService implements BaseDatabaseService { msg: `Acquired lock on database file '${this.userNamespace}.db'`, }); - // create metadata table - await tx.schema - .withSchema(this.userNamespace) - .createTable("ponder_metadata") - .addColumn("key", "text", (col) => col.primaryKey()) - .addColumn("value", "jsonb", (col) => col.notNull()) - .ifNotExists() - .execute(); - await createTables(); return { status: "success", checkpoint: zeroCheckpoint } as const; @@ -407,13 +418,6 @@ export class SqliteDatabaseService implements BaseDatabaseService { msg: `Acquired lock on schema '${this.userNamespace}' previously used by build '${previousBuildId}'`, }); - // clear metadata table - await tx - .withSchema(this.userNamespace) - // @ts-expect-error Kysely doesn't have types for user schema - .deleteFrom("ponder_metadata") - .execute(); - for (const tableName of Object.keys(previousSchema.tables)) { const tableId = hash([ this.userNamespace, diff --git a/packages/core/src/indexing-store/metadata.ts b/packages/core/src/indexing-store/metadata.ts index 141d879df..b4b9bf1ed 100644 --- a/packages/core/src/indexing-store/metadata.ts +++ b/packages/core/src/indexing-store/metadata.ts @@ -20,11 +20,11 @@ export const getMetadataStore = ({ .where("key", "=", "status") .executeTakeFirst(); - if (metadata === undefined) return undefined; + if (metadata!.value === null) return null; return encoding === "sqlite" - ? (JSON.parse(metadata.value) as Status) - : (metadata.value as Status); + ? (JSON.parse(metadata!.value) as Status) + : (metadata!.value as Status); }); }, setStatus: (status: Status) => { diff --git a/packages/core/src/indexing-store/store.ts b/packages/core/src/indexing-store/store.ts index b641323d1..35128a788 100644 --- a/packages/core/src/indexing-store/store.ts +++ b/packages/core/src/indexing-store/store.ts @@ -109,7 +109,7 @@ export type Status = { export type MetadataStore = { setStatus: (status: Status) => Promise; - getStatus: () => Promise; + getStatus: () => Promise; }; export type IndexingStore< diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index feb23e7f0..7fc0cef91 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -124,7 +124,7 @@ export async function createServer({ const status = await metadataStore.getStatus(); if ( - status !== undefined && + status !== null && Object.values(status).every(({ ready }) => ready === true) ) { return c.text("", 200); @@ -149,7 +149,7 @@ export async function createServer({ .post("/graphql", async (c) => { const status = await metadataStore.getStatus(); if ( - status === undefined || + status === null || Object.values(status).some(({ ready }) => ready === false) ) { return c.json( From aaf9d2fd1904d46882ef19b277ded9144840d5ab Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 10 Jul 2024 16:29:49 -0400 Subject: [PATCH 071/122] fix tests --- packages/core/src/indexing-store/metadata.test.ts | 2 +- packages/core/src/server/graphql/buildGraphqlSchema.test.ts | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/core/src/indexing-store/metadata.test.ts b/packages/core/src/indexing-store/metadata.test.ts index 455300a2e..1d8aa065f 100644 --- a/packages/core/src/indexing-store/metadata.test.ts +++ b/packages/core/src/indexing-store/metadata.test.ts @@ -27,7 +27,7 @@ test("getMetadata() empty", async (context) => { const status = await metadataStore.getStatus(); - expect(status).toBeUndefined(); + expect(status).toBe(null); await cleanup(); }); diff --git a/packages/core/src/server/graphql/buildGraphqlSchema.test.ts b/packages/core/src/server/graphql/buildGraphqlSchema.test.ts index f1a0298d8..b9828a053 100644 --- a/packages/core/src/server/graphql/buildGraphqlSchema.test.ts +++ b/packages/core/src/server/graphql/buildGraphqlSchema.test.ts @@ -3088,9 +3088,5 @@ test("metadata", async (context) => { }, }); - console.log(JSON.stringify(result, null, 2)); - - // console.log(result.data._metadata); - await cleanup(); }); From bdd94cd4862b487eb5d1ce1a71fbdf4359b02458 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 11 Jul 2024 10:58:05 -0400 Subject: [PATCH 072/122] cleanup virtual modules --- examples/reference-erc20/ponder-env.d.ts | 20 ---------- examples/reference-erc20/src/api/index.ts | 3 +- packages/core/package.json | 4 -- packages/core/src/build/plugin.ts | 46 ----------------------- packages/core/src/common/codegen.ts | 20 ---------- packages/core/src/drizzle/runtime.test.ts | 2 +- packages/core/src/drizzle/table.test-d.ts | 2 +- packages/core/src/drizzle/virtual.ts | 32 ---------------- packages/core/src/index.ts | 34 +++++++++++++++-- 9 files changed, 34 insertions(+), 129 deletions(-) delete mode 100644 packages/core/src/drizzle/virtual.ts diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index 2f78e46d0..f8e7347cf 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -25,23 +25,3 @@ declare module "@/generated" { Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; } - -declare module "ponder:db" { - import type { ConvertToDrizzleTable } from "@ponder/core"; - - type schema = typeof import("./ponder.schema.ts").default; - - const drizzleTables: { - [tableName in keyof schema]: ConvertToDrizzleTable< - tableName, - schema[tableName]["table"], - schema - >; - }; - - export = drizzleTables; -} - -declare module "ponder:db" { - export * from "@ponder/core/drizzle"; -} diff --git a/examples/reference-erc20/src/api/index.ts b/examples/reference-erc20/src/api/index.ts index e07257726..114868a35 100644 --- a/examples/reference-erc20/src/api/index.ts +++ b/examples/reference-erc20/src/api/index.ts @@ -1,6 +1,5 @@ -import { Account, desc } from "ponder:db"; import { ponder } from "@/generated"; -import { graphql } from "@ponder/core"; +import { desc, graphql } from "@ponder/core"; import { formatEther } from "viem"; // write file diff --git a/packages/core/package.json b/packages/core/package.json index 03f95e214..7a34f949b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -26,10 +26,6 @@ ".": { "import": "./dist/index.js", "types": "./dist/index.d.ts" - }, - "./drizzle": { - "import": "./dist/drizzle/virtual.js", - "types": "./dist/drizzle/virtual.d.ts" } }, "scripts": { diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 6ea651f63..5669ce4d3 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -31,52 +31,6 @@ export function replaceStateless(code: string, regex: RegExp, shim: string) { export const vitePluginPonder = (): Plugin => { return { name: "ponder", - load: (id) => { - if (id === "ponder:db") { - return `import schema from "ponder.schema"; -import config from "ponder.config"; -import { convertToDrizzleTable } from "@ponder/core"; -let databaseConfig = undefined -let envSchema = undefined; -let envPublishSchema = undefined; -if (process.env.RAILWAY_DEPLOYMENT_ID && process.env.RAILWAY_SERVICE_NAME) { - envSchema = \`\${process.env.RAILWAY_SERVICE_NAME}_\${process.env.RAILWAY_DEPLOYMENT_ID.slice( - 0, - 8, - )}\`; - envPublishSchema = "public"; -} else { - envSchema = "public"; -} -if (config.database?.kind) { - if (config.database.kind === "postgres") { - const schema = config.database.schema ?? envSchema; - const publishSchema = config.database.publishSchema ?? envPublishSchema; - databaseConfig = { kind: "postgres", schema, publishSchema }; - } else { - databaseConfig = { kind: "sqlite" }; - } -} else { - if (process.env.DATABASE_PRIVATE_URL || process.env.DATABASE_URL) { - const schema = envSchema; - const publishSchema = envPublishSchema; - databaseConfig = { kind: "postgres", schema, publishSchema }; - } else { - databaseConfig = { kind: "sqlite" }; - } -} -let drizzleTables = Object.fromEntries( - Object.entries(schema).map(([tableName, table]) => [ - tableName, - convertToDrizzleTable(tableName, table.table, databaseConfig), - ]), -); -export * from "@ponder/core/virtual"; -module.exports = drizzleTables; -`; - } - return null; - }, transform: (code, id) => { if (ponderRegex.test(code)) { const s = replaceStateless(code, ponderRegex, shim); diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index 100a8347d..6fe66b47a 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -30,26 +30,6 @@ declare module "@/generated" { Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; } - -declare module "ponder:db" { - import type { ConvertToDrizzleTable } from "@ponder/core"; - - type schema = typeof import("./ponder.schema.ts").default; - - const drizzleTables: { - [tableName in keyof schema]: ConvertToDrizzleTable< - tableName, - schema[tableName]["table"], - schema - >; - }; - - export = drizzleTables; -} - -declare module "ponder:db" { - export * from "@ponder/core/drizzle"; -} `; export function runCodegen({ diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index 5a52f0bb0..79980b0f3 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -7,10 +7,10 @@ import type { DatabaseService } from "@/database/service.js"; import { SqliteDatabaseService } from "@/database/sqlite/service.js"; import type { HistoricalStore } from "@/indexing-store/store.js"; import { createSchema } from "@/schema/schema.js"; +import { eq } from "drizzle-orm"; import { beforeEach, expect, test } from "vitest"; import type { DrizzleDb } from "./db.js"; import { convertToDrizzleTable, createDrizzleDb } from "./runtime.js"; -import { eq } from "./virtual.js"; beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); diff --git a/packages/core/src/drizzle/table.test-d.ts b/packages/core/src/drizzle/table.test-d.ts index 83e8f4774..7e1f6aa0c 100644 --- a/packages/core/src/drizzle/table.test-d.ts +++ b/packages/core/src/drizzle/table.test-d.ts @@ -1,9 +1,9 @@ import { createSchema } from "@/index.js"; +import { eq } from "drizzle-orm"; import type { Hex } from "viem"; import { expectTypeOf, test } from "vitest"; import type { DrizzleDb } from "./db.js"; import type { ConvertToDrizzleTable } from "./table.js"; -import { eq } from "./virtual.js"; test("select query promise", async () => { const schema = createSchema((p) => ({ diff --git a/packages/core/src/drizzle/virtual.ts b/packages/core/src/drizzle/virtual.ts deleted file mode 100644 index fa668802a..000000000 --- a/packages/core/src/drizzle/virtual.ts +++ /dev/null @@ -1,32 +0,0 @@ -export { - sql, - eq, - gt, - gte, - lt, - lte, - ne, - isNull, - isNotNull, - inArray, - notInArray, - exists, - notExists, - between, - notBetween, - like, - notIlike, - not, - asc, - desc, - and, - or, - count, - countDistinct, - avg, - avgDistinct, - sum, - sumDistinct, - max, - min, -} from "drizzle-orm"; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e686f7dd6..95814f9c3 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -22,6 +22,34 @@ export type DatabaseConfig = Prettify; export { graphql } from "@/graphql/index.js"; export { - type ConvertToDrizzleTable, - convertToDrizzleTable, -} from "@/drizzle/index.js"; + sql, + eq, + gt, + gte, + lt, + lte, + ne, + isNull, + isNotNull, + inArray, + notInArray, + exists, + notExists, + between, + notBetween, + like, + notIlike, + not, + asc, + desc, + and, + or, + count, + countDistinct, + avg, + avgDistinct, + sum, + sumDistinct, + max, + min, +} from "drizzle-orm"; From fb97142fc0da4b6f189142b7856d252a56111a71 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 11 Jul 2024 15:52:17 -0400 Subject: [PATCH 073/122] ponder.get --- packages/core/src/build/plugin.ts | 16 +- packages/core/src/hono/context.ts | 45 ++ packages/core/src/hono/handler.test-d.ts | 0 packages/core/src/hono/handler.ts | 533 +++++++++++++++++++++++ packages/core/src/hono/index.test.ts | 44 ++ packages/core/src/hono/index.ts | 37 ++ packages/core/src/types/virtual.ts | 10 +- 7 files changed, 675 insertions(+), 10 deletions(-) create mode 100644 packages/core/src/hono/context.ts create mode 100644 packages/core/src/hono/handler.test-d.ts create mode 100644 packages/core/src/hono/handler.ts create mode 100644 packages/core/src/hono/index.test.ts create mode 100644 packages/core/src/hono/index.ts diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 5669ce4d3..7bdc2ec6a 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -5,12 +5,18 @@ export const ponderRegex = /^import\s+\{[^}]*\bponder\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; export const shim = `import { Hono } from "hono"; -let __hono__ = new Hono(); + +let __ponderHono = { + handlers: [], + get(...maybePathOrHandlers) { + this.handlers.push(maybePathOrHandlers); + return this; + } +} + export let ponder = { - hono: __hono__, - get: __hono__.get, - post: __hono__.get, - use: __hono__.use, + hono: new Hono(), + ...__ponderHono, fns: [], on(name, fn) { this.fns.push({ name, fn }); diff --git a/packages/core/src/hono/context.ts b/packages/core/src/hono/context.ts new file mode 100644 index 000000000..a172e9a85 --- /dev/null +++ b/packages/core/src/hono/context.ts @@ -0,0 +1,45 @@ +import type { DrizzleDb } from "@/drizzle/db.js"; +import type { Env, Context as HonoContext, Input } from "hono"; + +export type Context = { + /** + * ... + */ + db: DrizzleDb; + /** + * Hono request object. + * + * @see https://hono.dev/docs/api/context#req + */ + req: HonoContext["req"]; + /** + * Hono response object. + * + * @see https://hono.dev/docs/api/context#res + */ + res: HonoContext["req"]; + /** + * Return the HTTP response. + * + * @see https://hono.dev/docs/api/context#body + */ + body: HonoContext["body"]; + /** + * Render text as `Content-Type:text/plain`. + * + * @see https://hono.dev/docs/api/context#text + */ + text: HonoContext["text"]; + /** + * Render JSON as `Content-Type:application/json`. + * + * @see https://hono.dev/docs/api/context#json + */ + json: HonoContext["json"]; + /** + * Hono redirect. + * + * @see https://hono.dev/docs/api/context#redirect + */ + redirect: HonoContext["redirect"]; +}; diff --git a/packages/core/src/hono/handler.test-d.ts b/packages/core/src/hono/handler.test-d.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/core/src/hono/handler.ts b/packages/core/src/hono/handler.ts new file mode 100644 index 000000000..3958a854d --- /dev/null +++ b/packages/core/src/hono/handler.ts @@ -0,0 +1,533 @@ +import type { Hono } from "hono"; +import type { + BlankInput, + BlankSchema, + Env, + HandlerResponse, + Input, + Next, +} from "hono/types"; +import type { Context } from "./context.js"; + +export type Handler< + path extends string = any, + input extends Input = BlankInput, + response extends HandlerResponse = any, +> = (c: Context) => response; + +export type MiddlewareHandler< + path extends string = string, + input extends Input = {}, +> = (c: Context, next: Next) => Promise; + +export type H< + path extends string = any, + input extends Input = BlankInput, + response extends HandlerResponse = any, +> = Handler | MiddlewareHandler; + +type BasePath = "/"; + +export type HandlerInterface = { + // app.get(handler) + < + path extends string = BasePath, + input extends Input = BlankInput, + response extends HandlerResponse = any, + >( + handler: Handler, + ): Hono; + + // app.get(handler x2) + < + path extends string = BasePath, + input extends Input = BlankInput, + input2 extends Input = input, + response extends HandlerResponse = any, + >( + ...handlers: [Handler, Handler] + ): Hono; + + // app.get(path, handler) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + >( + path: path, + handler: Handler, + ): Hono; + + // app.get(handler x 3) + < + path extends string = BasePath, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + >( + ...handlers: [ + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(path, handler x2) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + >( + path: path, + ...handlers: [Handler, Handler] + ): Hono; + + // app.get(handler x 4) + < + path extends string = BasePath, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + >( + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(path, handler x3) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + >( + path: path, + ...handlers: [ + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(handler x 5) + < + path extends string = BasePath, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + >( + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(path, handler x4) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + >( + path: path, + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(handler x 6) + < + path extends string = BasePath, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + >( + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(path, handler x5) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + >( + path: path, + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(handler x 7) + < + path extends string = BasePath, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + input7 extends Input = input & input2 & input3 & input4 & input5 & input6, + >( + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(path, handler x6) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + >( + path: path, + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(handler x 8) + < + path extends string = BasePath, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + input7 extends Input = input & input2 & input3 & input4 & input5 & input6, + input8 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7, + >( + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(path, handler x7) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + input7 extends Input = input & input2 & input3 & input4 & input5 & input6, + >( + path: path, + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(handler x 9) + < + path extends string = BasePath, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + input7 extends Input = input & input2 & input3 & input4 & input5 & input6, + input8 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7, + input9 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7 & + input8, + >( + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(path, handler x8) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + input7 extends Input = input & input2 & input3 & input4 & input5 & input6, + input8 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7, + >( + path: path, + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(handler x 10) + < + path extends string = BasePath, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + input7 extends Input = input & input2 & input3 & input4 & input5 & input6, + input8 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7, + input9 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7 & + input8, + input10 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7 & + input8 & + input9, + >( + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(path, handler x9) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + input7 extends Input = input & input2 & input3 & input4 & input5 & input6, + input8 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7, + input9 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7 & + input8, + >( + path: path, + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(path, handler x10) + < + path extends string, + response extends HandlerResponse = any, + input extends Input = BlankInput, + input2 extends Input = input, + input3 extends Input = input & input2, + input4 extends Input = input & input2 & input3, + input5 extends Input = input & input2 & input3 & input4, + input6 extends Input = input & input2 & input3 & input4 & input5, + input7 extends Input = input & input2 & input3 & input4 & input5 & input6, + input8 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7, + input9 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7 & + input8, + input10 extends Input = input & + input2 & + input3 & + input4 & + input5 & + input6 & + input7 & + input8 & + input9, + >( + path: path, + ...handlers: [ + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + Handler, + ] + ): Hono; + + // app.get(...handlers[]) + < + path extends string = BasePath, + input extends Input = BlankInput, + response extends HandlerResponse = any, + >( + ...handlers: Handler[] + ): Hono; + + // app.get(path, ...handlers[]) + < + path extends string, + input extends Input = BlankInput, + response extends HandlerResponse = any, + >( + path: path, + ...handlers: Handler[] + ): Hono; + + // app.get(path) + (path: path): Hono; +}; diff --git a/packages/core/src/hono/index.test.ts b/packages/core/src/hono/index.test.ts new file mode 100644 index 000000000..73e466421 --- /dev/null +++ b/packages/core/src/hono/index.test.ts @@ -0,0 +1,44 @@ +import { Hono } from "hono"; +import { expect, test } from "vitest"; +import type { Handler } from "./handler.js"; +import { applyPathOrHandlers } from "./index.js"; + +type MockPonderHono = { + handlers: Parameters["1"]; + get: ( + maybePathOrHandler: string | Handler, + ...handlers: Handler[] + ) => MockPonderHono; +}; + +const getMockPonderHono = (): MockPonderHono => ({ + handlers: [], + get(..._handlers) { + this.handlers.push(_handlers); + return this; + }, +}); + +test("get request w/o path", async () => { + const ponderHono = getMockPonderHono().get((c) => { + return c.text("hi"); + }); + + const hono = applyPathOrHandlers(new Hono(), ponderHono.handlers); + + const response = await hono.request(""); + expect(await response.text()).toBe("hi"); +}); + +test("get request w/ path", async () => { + const ponderHono = getMockPonderHono().get("/hi", (c) => { + return c.text("hi"); + }); + + const hono = applyPathOrHandlers(new Hono(), ponderHono.handlers); + + const response = await hono.request("/hi"); + expect(await response.text()).toBe("hi"); +}); + +test.todo("get request with multiple handlers"); diff --git a/packages/core/src/hono/index.ts b/packages/core/src/hono/index.ts new file mode 100644 index 000000000..c978b2dfb --- /dev/null +++ b/packages/core/src/hono/index.ts @@ -0,0 +1,37 @@ +import type { Hono } from "hono"; +import type { Handler } from "./handler.js"; + +export const applyPathOrHandlers = ( + hono: Hono, + pathOrHandlers: [ + maybePathOrHandler: string | Handler, + ...handlers: Handler[], + ][], +) => { + // add custom properties to hono context + const addCustomContext = (handler: Handler) => (c: any) => { + return handler(c); + }; + + // register collected path + handlers to the underlying hono instance + // from https://github.com/honojs/hono/blob/main/src/hono-base.ts#L125-L142 + for (const [maybePathOrHandler, ...handlers] of pathOrHandlers) { + let path = "/"; + + if (typeof maybePathOrHandler === "string") { + path = maybePathOrHandler; + } else { + // @ts-expect-error access private property + hono.addRoute("get", path, addCustomContext(maybePathOrHandler)); + } + + for (const handler of handlers) { + if (typeof handler !== "string") { + // @ts-expect-error access private property + hono.addRoute("get", path, addCustomContext(handler)); + } + } + } + + return hono; +}; diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index 7b5978e80..e284ad60f 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -6,7 +6,7 @@ import type { SafeEventNames, SafeFunctionNames, } from "@/config/utilityTypes.js"; -import type { DrizzleDb } from "@/drizzle/db.js"; +import type { HandlerInterface } from "@/hono/handler.js"; import type { ReadOnlyClient } from "@/indexing/ponderActions.js"; import type { Schema as BuilderSchema } from "@/schema/common.js"; import type { InferSchemaType } from "@/schema/infer.js"; @@ -248,9 +248,9 @@ export namespace Virtual { }, ) => Promise | void, ) => void; - get: Hono<{ Variables: { db: DrizzleDb } }>["get"]; - post: Hono<{ Variables: { db: DrizzleDb } }>["post"]; - use: Hono<{ Variables: { db: DrizzleDb } }>["use"]; - hono: Hono<{ Variables: { db: DrizzleDb } }>; + get: HandlerInterface; + post: HandlerInterface; + use: HandlerInterface; + hono: Hono; }; } From ea2e215968ccd2aea99d577fd2e42c497a27da77 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 11 Jul 2024 18:38:18 -0400 Subject: [PATCH 074/122] ponder.use --- packages/core/src/bin/commands/serve.ts | 8 +- packages/core/src/bin/utils/run.ts | 4 - packages/core/src/build/plugin.ts | 14 +- packages/core/src/build/service.ts | 12 +- .../core/src/database/postgres/service.ts | 2 +- packages/core/src/database/sqlite/service.ts | 5 +- packages/core/src/drizzle/runtime.ts | 2 + packages/core/src/hono/context.ts | 10 + packages/core/src/hono/handler.ts | 470 +++++++++++++----- packages/core/src/hono/index.test.ts | 76 ++- packages/core/src/hono/index.ts | 64 ++- packages/core/src/indexing-store/readonly.ts | 2 +- packages/core/src/server/service.ts | 6 - packages/core/src/types/hono.ts | 12 + packages/core/src/types/virtual.ts | 9 +- 15 files changed, 502 insertions(+), 194 deletions(-) create mode 100644 packages/core/src/types/hono.ts diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index c014c1c6e..7b1da1ab3 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -5,7 +5,6 @@ import { MetricsService } from "@/common/metrics.js"; import { buildOptions } from "@/common/options.js"; import { buildPayload, createTelemetry } from "@/common/telemetry.js"; import { PostgresDatabaseService } from "@/database/postgres/service.js"; -import type { NamespaceInfo } from "@/database/service.js"; import { getReadonlyStore } from "@/indexing-store/readonly.js"; import { createServer } from "@/server/service.js"; import type { CliOptions } from "../ponder.js"; @@ -99,12 +98,10 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { const readonlyStore = getReadonlyStore({ encoding: "postgres", schema, - // Note: `ponder serve` serves data from the `publishSchema`. Also, it does - // not need the other fields in NamespaceInfo because it only uses findUnique - // and findMany. We should ultimately add a PublicStore interface for this. + // Note: `ponder serve` serves data from the `publishSchema`. namespaceInfo: { userNamespace: databaseConfig.publishSchema, - } as unknown as NamespaceInfo, + }, db: database.readonlyDb, common, }); @@ -113,7 +110,6 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { app, readonlyStore, schema, - database: { kind: "postgres", pool: database.readonlyPool }, common, }); server.setHealthy(); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index f753a2bc2..a7db7b556 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -106,10 +106,6 @@ export async function run({ app: build.app, readonlyStore, schema, - database: - database.kind === "sqlite" - ? { kind: "sqlite", database: database.userDatabase } - : { kind: "postgres", pool: database.readonlyPool }, common, }); diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 7bdc2ec6a..353ac87f5 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -7,11 +7,19 @@ export const ponderRegex = export const shim = `import { Hono } from "hono"; let __ponderHono = { - handlers: [], + routes: [], get(...maybePathOrHandlers) { - this.handlers.push(maybePathOrHandlers); + this.routes.push({method: "GET", pathOrHandlers: maybePathOrHandlers}); return this; - } + }, + post(...maybePathOrHandlers) { + this.routes.push({method: "POST", pathOrHandlers: maybePathOrHandlers}); + return this; + }, + use(...maybePathOrHandlers) { + this.routes.push({method: "USE", pathOrHandlers: maybePathOrHandlers}); + return this; + } } export let ponder = { diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 53c3f208f..1b45860ee 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -7,6 +7,7 @@ import type { DatabaseConfig } from "@/config/database.js"; import type { Network } from "@/config/networks.js"; import type { EventSource } from "@/config/sources.js"; import { buildGraphQLSchema } from "@/graphql/buildGraphqlSchema.js"; +import type { PonderRoutes } from "@/hono/index.js"; import type { Schema } from "@/schema/common.js"; import { glob } from "glob"; import type { GraphQLSchema } from "graphql"; @@ -54,6 +55,7 @@ export type Build = { indexingFunctions: IndexingFunctions; // Server app?: Hono; + routes?: PonderRoutes; }; export type BuildResult = @@ -67,7 +69,7 @@ type RawBuild = { indexingFunctions: RawIndexingFunctions; contentHash: string; }; - server: { app?: Hono }; + server: { app?: Hono; routes?: PonderRoutes }; }; export const create = async ({ @@ -432,6 +434,7 @@ const executeServer = async ( | { status: "success"; app?: Hono; + routes?: PonderRoutes; } | { status: "error"; error: Error } > => { @@ -458,7 +461,11 @@ const executeServer = async ( return executeResult; } - return { status: "success", app: executeResult.exports?.ponder?.hono }; + return { + status: "success", + app: executeResult.exports?.ponder?.hono, + routes: executeResult.exports?.ponder?.routes, + }; }; const validateAndBuild = async ( @@ -532,6 +539,7 @@ const validateAndBuild = async ( indexingFunctions: buildConfigAndIndexingFunctionsResult.indexingFunctions, app: rawBuild.server.app, + routes: rawBuild.server.routes, }, }; }; diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index afc56394b..6f8c749fa 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -67,7 +67,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { private internalPool: Pool; private syncPool: Pool; private indexingPool: Pool; - readonlyPool: Pool; + private readonlyPool: Pool; constructor({ common, diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index e644652c9..058e0945a 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -51,8 +51,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { private userNamespace: string; private internalNamespace: string; - internalDatabase: SqliteDatabase; - userDatabase: SqliteDatabase; + private internalDatabase: SqliteDatabase; private syncDatabase: SqliteDatabase; db: HeadlessKysely; @@ -92,8 +91,6 @@ export class SqliteDatabaseService implements BaseDatabaseService { `ATTACH DATABASE '${userDatabaseFile}' AS ${this.userNamespace}`, ); - this.userDatabase = createSqliteDatabase(userDatabaseFile); - this.db = new HeadlessKysely({ name: "internal", common, diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index 51110f220..b19c7a5bf 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -31,6 +31,8 @@ import { SQLiteBigintBuilder } from "./bigint.js"; import { PgHexBuilder, SQLiteHexBuilder } from "./hex.js"; import { SQLiteJsonBuilder } from "./json.js"; +// TODO(ask kevin for help on this) +// is what database objects are stateful vs stateless? export const createDrizzleDb = ( database: | { kind: "postgres"; pool: Pool } diff --git a/packages/core/src/hono/context.ts b/packages/core/src/hono/context.ts index a172e9a85..f1ba6e94a 100644 --- a/packages/core/src/hono/context.ts +++ b/packages/core/src/hono/context.ts @@ -43,3 +43,13 @@ export type Context = { */ redirect: HonoContext["redirect"]; }; + +export type MiddlewareContext< + path extends string = string, + input extends Input = {}, +> = HonoContext & { + /** + * ... + */ + db: DrizzleDb; +}; diff --git a/packages/core/src/hono/handler.ts b/packages/core/src/hono/handler.ts index 3958a854d..da1571f8c 100644 --- a/packages/core/src/hono/handler.ts +++ b/packages/core/src/hono/handler.ts @@ -1,13 +1,6 @@ -import type { Hono } from "hono"; -import type { - BlankInput, - BlankSchema, - Env, - HandlerResponse, - Input, - Next, -} from "hono/types"; -import type { Context } from "./context.js"; +import type { PonderHono } from "@/types/hono.js"; +import type { BlankInput, HandlerResponse, Input, Next } from "hono/types"; +import type { Context, MiddlewareContext } from "./context.js"; export type Handler< path extends string = any, @@ -18,13 +11,7 @@ export type Handler< export type MiddlewareHandler< path extends string = string, input extends Input = {}, -> = (c: Context, next: Next) => Promise; - -export type H< - path extends string = any, - input extends Input = BlankInput, - response extends HandlerResponse = any, -> = Handler | MiddlewareHandler; +> = (c: MiddlewareContext, next: Next) => Promise; type BasePath = "/"; @@ -36,7 +23,7 @@ export type HandlerInterface = { response extends HandlerResponse = any, >( handler: Handler, - ): Hono; + ): PonderHono; // app.get(handler x2) < @@ -46,7 +33,7 @@ export type HandlerInterface = { response extends HandlerResponse = any, >( ...handlers: [Handler, Handler] - ): Hono; + ): PonderHono; // app.get(path, handler) < @@ -56,7 +43,7 @@ export type HandlerInterface = { >( path: path, handler: Handler, - ): Hono; + ): PonderHono; // app.get(handler x 3) < @@ -67,11 +54,11 @@ export type HandlerInterface = { input3 extends Input = input & input2, >( ...handlers: [ - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(path, handler x2) < @@ -81,8 +68,11 @@ export type HandlerInterface = { input2 extends Input = input, >( path: path, - ...handlers: [Handler, Handler] - ): Hono; + ...handlers: [ + MiddlewareHandler, + Handler, + ] + ): PonderHono; // app.get(handler x 4) < @@ -94,12 +84,12 @@ export type HandlerInterface = { input4 extends Input = input & input2 & input3, >( ...handlers: [ - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(path, handler x3) < @@ -111,11 +101,11 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(handler x 5) < @@ -128,13 +118,13 @@ export type HandlerInterface = { input5 extends Input = input & input2 & input3 & input4, >( ...handlers: [ - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(path, handler x4) < @@ -147,12 +137,12 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(handler x 6) < @@ -166,14 +156,14 @@ export type HandlerInterface = { input6 extends Input = input & input2 & input3 & input4 & input5, >( ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(path, handler x5) < @@ -187,13 +177,13 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(handler x 7) < @@ -208,15 +198,15 @@ export type HandlerInterface = { input7 extends Input = input & input2 & input3 & input4 & input5 & input6, >( ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(path, handler x6) < @@ -231,14 +221,14 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(handler x 8) < @@ -260,16 +250,16 @@ export type HandlerInterface = { input7, >( ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(path, handler x7) < @@ -285,15 +275,15 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(handler x 9) < @@ -323,17 +313,17 @@ export type HandlerInterface = { input8, >( ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(path, handler x8) < @@ -356,16 +346,16 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(handler x 10) < @@ -404,18 +394,18 @@ export type HandlerInterface = { input9, >( ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(path, handler x9) < @@ -446,17 +436,17 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(path, handler x10) < @@ -496,18 +486,18 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, Handler, ] - ): Hono; + ): PonderHono; // app.get(...handlers[]) < @@ -516,7 +506,7 @@ export type HandlerInterface = { response extends HandlerResponse = any, >( ...handlers: Handler[] - ): Hono; + ): PonderHono; // app.get(path, ...handlers[]) < @@ -526,8 +516,230 @@ export type HandlerInterface = { >( path: path, ...handlers: Handler[] - ): Hono; + ): PonderHono; // app.get(path) - (path: path): Hono; + (path: path): PonderHono; }; + +export interface MiddlewareHandlerInterface { + //// app.use(...handlers[]) + (...handlers: MiddlewareHandler[]): PonderHono; + + // app.use(handler) + (handler: MiddlewareHandler): PonderHono; + + // app.use(handler x2) + ( + ...handlers: [MiddlewareHandler, MiddlewareHandler] + ): PonderHono; + + // app.get(path, handler) + ( + path: path, + handler: MiddlewareHandler, + ): PonderHono; + + // app.use(handler x3) + ( + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.get(path, handler x2) + ( + path: path, + ...handlers: [MiddlewareHandler, MiddlewareHandler] + ): PonderHono; + + // app.use(handler x4) + ( + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.get(path, handler x3) + ( + path: path, + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.use(handler x5) + ( + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.get(path, handler x4) + ( + path: path, + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.use(handler x6) + ( + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.get(path, handler x5) + ( + path: path, + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.use(handler x7) + ( + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.get(path, handler x6) + ( + path: path, + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.use(handler x8) + ( + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.get(path, handler x7) + ( + path: path, + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.use(handler x9) + ( + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.get(path, handler x8) + ( + path: path, + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.use(handler x10) + ( + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + // app.get(path, handler x9) + ( + path: path, + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; + + //// app.use(path, ...handlers[]) + ( + path: path, + ...handlers: MiddlewareHandler[] + ): PonderHono; +} diff --git a/packages/core/src/hono/index.test.ts b/packages/core/src/hono/index.test.ts index 73e466421..021caeed2 100644 --- a/packages/core/src/hono/index.test.ts +++ b/packages/core/src/hono/index.test.ts @@ -1,44 +1,102 @@ import { Hono } from "hono"; import { expect, test } from "vitest"; import type { Handler } from "./handler.js"; -import { applyPathOrHandlers } from "./index.js"; +import { type PonderRoutes, applyHonoRoutes } from "./index.js"; type MockPonderHono = { - handlers: Parameters["1"]; + routes: PonderRoutes; get: ( maybePathOrHandler: string | Handler, ...handlers: Handler[] ) => MockPonderHono; + post: ( + maybePathOrHandler: string | Handler, + ...handlers: Handler[] + ) => MockPonderHono; + use: ( + maybePathOrHandler: string | Handler, + ...handlers: Handler[] + ) => MockPonderHono; }; const getMockPonderHono = (): MockPonderHono => ({ - handlers: [], + routes: [], get(..._handlers) { - this.handlers.push(_handlers); + this.routes.push({ method: "GET", pathOrHandlers: _handlers }); + return this; + }, + post(..._handlers) { + this.routes.push({ method: "POST", pathOrHandlers: _handlers }); + return this; + }, + use(..._handlers) { + this.routes.push({ method: "USE", pathOrHandlers: _handlers }); return this; }, }); -test("get request w/o path", async () => { +test("get() w/o path", async () => { const ponderHono = getMockPonderHono().get((c) => { return c.text("hi"); }); - const hono = applyPathOrHandlers(new Hono(), ponderHono.handlers); + const hono = applyHonoRoutes(new Hono(), ponderHono.routes); const response = await hono.request(""); expect(await response.text()).toBe("hi"); }); -test("get request w/ path", async () => { +test("get() w/ path", async () => { const ponderHono = getMockPonderHono().get("/hi", (c) => { return c.text("hi"); }); - const hono = applyPathOrHandlers(new Hono(), ponderHono.handlers); + const hono = applyHonoRoutes(new Hono(), ponderHono.routes); + + const response = await hono.request("/hi"); + expect(await response.text()).toBe("hi"); +}); + +test("get() w/ middlware", async () => { + const ponderHono = getMockPonderHono().get( + "/hi", + // @ts-ignore + (c, next) => { + next(); + }, + (c) => { + return c.text("hi"); + }, + ); + + const hono = applyHonoRoutes(new Hono(), ponderHono.routes); const response = await hono.request("/hi"); expect(await response.text()).toBe("hi"); }); -test.todo("get request with multiple handlers"); +test("use() w/o path", async () => { + // @ts-ignore + const ponderHono = getMockPonderHono().use((c, next) => { + next(); + return c.text("hi"); + }); + + const hono = applyHonoRoutes(new Hono(), ponderHono.routes); + + const response = await hono.request(""); + expect(await response.text()).toBe("hi"); +}); + +test("use() w/ path", async () => { + // @ts-ignore + const ponderHono = getMockPonderHono().use("/hi", (c, next) => { + next(); + return c.text("hi"); + }); + + const hono = applyHonoRoutes(new Hono(), ponderHono.routes); + + const response = await hono.request("/hi"); + expect(await response.text()).toBe("hi"); +}); diff --git a/packages/core/src/hono/index.ts b/packages/core/src/hono/index.ts index c978b2dfb..a824a1371 100644 --- a/packages/core/src/hono/index.ts +++ b/packages/core/src/hono/index.ts @@ -1,34 +1,54 @@ import type { Hono } from "hono"; -import type { Handler } from "./handler.js"; +import type { Handler, MiddlewareHandler } from "./handler.js"; -export const applyPathOrHandlers = ( - hono: Hono, +export type PonderRoutes = { + method: "GET" | "POST" | "USE"; pathOrHandlers: [ - maybePathOrHandler: string | Handler, - ...handlers: Handler[], - ][], -) => { + maybePathOrHandler: string | Handler | MiddlewareHandler, + ...handlers: (Handler | MiddlewareHandler)[], + ]; +}[]; + +export const applyHonoRoutes = (hono: Hono, routes: PonderRoutes) => { // add custom properties to hono context - const addCustomContext = (handler: Handler) => (c: any) => { - return handler(c); - }; + const addCustomContext = + (handler: Handler | MiddlewareHandler) => (c: any, next: any) => { + return handler(c, next); + }; - // register collected path + handlers to the underlying hono instance - // from https://github.com/honojs/hono/blob/main/src/hono-base.ts#L125-L142 - for (const [maybePathOrHandler, ...handlers] of pathOrHandlers) { + for (const { + method, + pathOrHandlers: [maybePathOrHandler, ...handlers], + } of routes) { let path = "/"; + if (method === "GET" || method === "POST") { + // register collected "GET" or "POST" path + handlers to the underlying hono instance + // from https://github.com/honojs/hono/blob/main/src/hono-base.ts#L125-L142 + if (typeof maybePathOrHandler === "string") { + path = maybePathOrHandler; + } else { + // @ts-expect-error access private property + hono.addRoute(method, path, addCustomContext(maybePathOrHandler)); + } - if (typeof maybePathOrHandler === "string") { - path = maybePathOrHandler; + for (const handler of handlers) { + if (typeof handler !== "string") { + // @ts-expect-error access private property + hono.addRoute(method, path, addCustomContext(handler)); + } + } } else { - // @ts-expect-error access private property - hono.addRoute("get", path, addCustomContext(maybePathOrHandler)); - } - - for (const handler of handlers) { - if (typeof handler !== "string") { + // register collected middleware to the underlying hono instance + // from: https://github.com/honojs/hono/blob/main/src/hono-base.ts#L158-L169 + if (typeof maybePathOrHandler === "string") { + path = maybePathOrHandler; + } else { + path = "*"; + handlers.unshift(maybePathOrHandler); + } + for (const handler of handlers) { // @ts-expect-error access private property - hono.addRoute("get", path, addCustomContext(handler)); + hono.addRoute("ALL", path, handler); } } } diff --git a/packages/core/src/indexing-store/readonly.ts b/packages/core/src/indexing-store/readonly.ts index 790c8d1bf..19c50cae2 100644 --- a/packages/core/src/indexing-store/readonly.ts +++ b/packages/core/src/indexing-store/readonly.ts @@ -29,7 +29,7 @@ export const getReadonlyStore = ({ }: { encoding: "sqlite" | "postgres"; schema: Schema; - namespaceInfo: NamespaceInfo; + namespaceInfo: Pick; db: HeadlessKysely; common: Common; }): ReadonlyStore => ({ diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 8270c4447..0f2b57cb8 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -3,14 +3,12 @@ import type { Common } from "@/common/common.js"; import { createDrizzleDb } from "@/drizzle/runtime.js"; import type { ReadonlyStore } from "@/indexing-store/store.js"; import type { Schema } from "@/schema/common.js"; -import type { SqliteDatabase } from "@/utils/sqlite.js"; import { startClock } from "@/utils/timer.js"; import { serve } from "@hono/node-server"; import { Hono } from "hono"; import { cors } from "hono/cors"; import { createMiddleware } from "hono/factory"; import { createHttpTerminator } from "http-terminator"; -import type { Pool } from "pg"; import { onError } from "./error.js"; type Server = { @@ -24,15 +22,11 @@ export async function createServer({ app: userApp, schema, readonlyStore, - database, common, }: { app?: Hono; schema: Schema; readonlyStore: ReadonlyStore; - database: - | { kind: "postgres"; pool: Pool } - | { kind: "sqlite"; database: SqliteDatabase }; common: Common; }): Promise { // Create hono app diff --git a/packages/core/src/types/hono.ts b/packages/core/src/types/hono.ts new file mode 100644 index 000000000..8a19a621c --- /dev/null +++ b/packages/core/src/types/hono.ts @@ -0,0 +1,12 @@ +import type { + HandlerInterface, + MiddlewareHandlerInterface, +} from "@/hono/handler.js"; +import type { Hono } from "hono"; + +export type PonderHono = { + get: HandlerInterface; + post: HandlerInterface; + use: MiddlewareHandlerInterface; + hono: Hono; +}; diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index e284ad60f..133bdaa29 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -6,7 +6,6 @@ import type { SafeEventNames, SafeFunctionNames, } from "@/config/utilityTypes.js"; -import type { HandlerInterface } from "@/hono/handler.js"; import type { ReadOnlyClient } from "@/indexing/ponderActions.js"; import type { Schema as BuilderSchema } from "@/schema/common.js"; import type { InferSchemaType } from "@/schema/infer.js"; @@ -18,7 +17,7 @@ import type { TransactionReceipt, } from "@/types/eth.js"; import type { DatabaseModel } from "@/types/model.js"; -import type { Hono } from "hono"; +import type { PonderHono } from "./hono.js"; import type { Prettify } from "./utils.js"; export namespace Virtual { @@ -248,9 +247,5 @@ export namespace Virtual { }, ) => Promise | void, ) => void; - get: HandlerInterface; - post: HandlerInterface; - use: HandlerInterface; - hono: Hono; - }; + } & PonderHono; } From 40ec5528b7aa7a85814605704e484e2bd1f0a066 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 11 Jul 2024 19:10:13 -0400 Subject: [PATCH 075/122] add db to hono context --- packages/core/src/bin/commands/serve.ts | 7 +++++-- packages/core/src/bin/utils/run.ts | 4 +++- .../core/src/database/postgres/service.ts | 2 +- packages/core/src/database/sqlite/service.ts | 6 ++++-- packages/core/src/drizzle/runtime.test.ts | 13 +------------ packages/core/src/drizzle/runtime.ts | 18 +++++++----------- packages/core/src/hono/index.ts | 8 ++++++-- packages/core/src/server/service.ts | 19 ++++++++++++------- 8 files changed, 39 insertions(+), 38 deletions(-) diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index 7b1da1ab3..5b68e3f28 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -65,7 +65,8 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { properties: { cli_command: "serve", ...buildPayload(initialResult.build) }, }); - const { databaseConfig, optionsConfig, schema, app } = initialResult.build; + const { databaseConfig, optionsConfig, schema, app, routes } = + initialResult.build; common.options = { ...common.options, ...optionsConfig }; @@ -108,9 +109,11 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { const server = await createServer({ app, + routes, + common, + database, readonlyStore, schema, - common, }); server.setHealthy(); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index a7db7b556..4cc58df28 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -104,9 +104,11 @@ export async function run({ const server = await createServer({ app: build.app, + routes: build.routes, + common, + database, readonlyStore, schema, - common, }); // This can be a long-running operation, so it's best to do it after diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index 6f8c749fa..afc56394b 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -67,7 +67,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { private internalPool: Pool; private syncPool: Pool; private indexingPool: Pool; - private readonlyPool: Pool; + readonlyPool: Pool; constructor({ common, diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index 058e0945a..11de070af 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -51,6 +51,8 @@ export class SqliteDatabaseService implements BaseDatabaseService { private userNamespace: string; private internalNamespace: string; + userDatabaseFile: string; + private internalDatabase: SqliteDatabase; private syncDatabase: SqliteDatabase; @@ -77,7 +79,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { this.deleteV3DatabaseFiles(); this.userNamespace = userNamespace; - const userDatabaseFile = path.join(directory, `${userNamespace}.db`); + this.userDatabaseFile = path.join(directory, `${userNamespace}.db`); // Note that SQLite supports using "main" as the schema name for tables // in the primary database (as opposed to attached databases). We include @@ -88,7 +90,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { this.internalDatabase = createSqliteDatabase(internalDatabaseFile); this.internalDatabase.exec( - `ATTACH DATABASE '${userDatabaseFile}' AS ${this.userNamespace}`, + `ATTACH DATABASE '${this.userDatabaseFile}' AS ${this.userNamespace}`, ); this.db = new HeadlessKysely({ diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index 79980b0f3..1be6b67a3 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -4,7 +4,6 @@ import { setupIsolatedDatabase, } from "@/_test/setup.js"; import type { DatabaseService } from "@/database/service.js"; -import { SqliteDatabaseService } from "@/database/sqlite/service.js"; import type { HistoricalStore } from "@/indexing-store/store.js"; import { createSchema } from "@/schema/schema.js"; import { eq } from "drizzle-orm"; @@ -16,17 +15,7 @@ beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); const createDb = (database: DatabaseService) => { - if (database instanceof SqliteDatabaseService) { - return createDrizzleDb({ - kind: database.kind, - database: database.userDatabase, - }) as unknown as DrizzleDb; - } else { - return createDrizzleDb({ - kind: database.kind, - pool: database.readonlyPool, - }) as unknown as DrizzleDb; - } + return createDrizzleDb(database) as unknown as DrizzleDb; }; test("runtime select", async (context) => { diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index b19c7a5bf..8af966440 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -1,4 +1,5 @@ import type { DatabaseConfig } from "@/config/database.js"; +import type { DatabaseService } from "@/database/service.js"; import type { Table as PonderTable } from "@/schema/common.js"; import { isEnumColumn, @@ -8,7 +9,7 @@ import { isReferenceColumn, isScalarColumn, } from "@/schema/utils.js"; -import type { SqliteDatabase } from "@/utils/sqlite.js"; +import { createSqliteDatabase } from "@/utils/sqlite.js"; import type { Table } from "drizzle-orm"; import { drizzle as drizzleSQLite } from "drizzle-orm/better-sqlite3"; import { drizzle as drizzlePg } from "drizzle-orm/node-postgres"; @@ -26,27 +27,22 @@ import { text as SQLiteText, sqliteTable, } from "drizzle-orm/sqlite-core"; -import type { Pool } from "pg"; import { SQLiteBigintBuilder } from "./bigint.js"; import { PgHexBuilder, SQLiteHexBuilder } from "./hex.js"; import { SQLiteJsonBuilder } from "./json.js"; -// TODO(ask kevin for help on this) -// is what database objects are stateful vs stateless? -export const createDrizzleDb = ( - database: - | { kind: "postgres"; pool: Pool } - | { kind: "sqlite"; database: SqliteDatabase }, -) => { +export const createDrizzleDb = (database: DatabaseService) => { if (database.kind === "postgres") { - const drizzle = drizzlePg(database.pool); + const drizzle = drizzlePg(database.readonlyPool); return { // @ts-ignore select: (...args: any[]) => drizzle.select(...args), execute: (query: any) => drizzle.execute(query), }; } else { - const drizzle = drizzleSQLite(database.database); + const drizzle = drizzleSQLite( + createSqliteDatabase(database.userDatabaseFile), + ); return { // @ts-ignore select: (...args: any[]) => drizzle.select(...args), diff --git a/packages/core/src/hono/index.ts b/packages/core/src/hono/index.ts index a824a1371..83f0a4a40 100644 --- a/packages/core/src/hono/index.ts +++ b/packages/core/src/hono/index.ts @@ -9,11 +9,15 @@ export type PonderRoutes = { ]; }[]; -export const applyHonoRoutes = (hono: Hono, routes: PonderRoutes) => { +export const applyHonoRoutes = ( + hono: Hono, + routes: PonderRoutes, + customContext?: object, +) => { // add custom properties to hono context const addCustomContext = (handler: Handler | MiddlewareHandler) => (c: any, next: any) => { - return handler(c, next); + return handler({ ...customContext, ...c }, next); }; for (const { diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 0f2b57cb8..a17397f04 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -1,6 +1,8 @@ import http from "node:http"; import type { Common } from "@/common/common.js"; +import type { DatabaseService } from "@/database/service.js"; import { createDrizzleDb } from "@/drizzle/runtime.js"; +import { type PonderRoutes, applyHonoRoutes } from "@/hono/index.js"; import type { ReadonlyStore } from "@/indexing-store/store.js"; import type { Schema } from "@/schema/common.js"; import { startClock } from "@/utils/timer.js"; @@ -20,14 +22,18 @@ type Server = { export async function createServer({ app: userApp, + routes: userRoutes, + common, + database, schema, readonlyStore, - common, }: { app?: Hono; + routes?: PonderRoutes; + common: Common; + database: DatabaseService; schema: Schema; readonlyStore: ReadonlyStore; - common: Common; }): Promise { // Create hono app @@ -102,7 +108,6 @@ export async function createServer({ const db = createDrizzleDb(database); const contextMiddleware = createMiddleware(async (c, next) => { - c.set("db", db); c.set("readonlyStore", readonlyStore); c.set("schema", schema); await next(); @@ -113,7 +118,7 @@ export async function createServer({ .route("/_ponder", ponderApp) .use(contextMiddleware); - if (userApp !== undefined) { + if (userApp !== undefined && userRoutes !== undefined) { for (const route of userApp.routes) { // Validate user routes don't conflict with ponder routes if (route.path.startsWith("/_ponder")) { @@ -130,12 +135,12 @@ export async function createServer({ .map((r) => r.path) .join(", ")}]`, }); - } - if (userApp !== undefined) { hono.route( "/", - userApp.onError((error, c) => onError(error, c, common)), + applyHonoRoutes(userApp, userRoutes, { db }).onError((error, c) => + onError(error, c, common), + ), ); } From ea520acbed7d9cb917db8a90128c1cf03f075d4c Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 11 Jul 2024 20:17:29 -0400 Subject: [PATCH 076/122] c.tables --- examples/reference-erc20/src/api/index.ts | 10 +- packages/core/src/drizzle/index.ts | 3 - packages/core/src/drizzle/runtime.ts | 94 ++-- packages/core/src/hono/context.ts | 31 +- packages/core/src/hono/handler.ts | 529 +++++++++++----------- packages/core/src/server/service.ts | 9 +- packages/core/src/types/hono.ts | 9 +- packages/core/src/types/virtual.ts | 2 +- 8 files changed, 374 insertions(+), 313 deletions(-) delete mode 100644 packages/core/src/drizzle/index.ts diff --git a/examples/reference-erc20/src/api/index.ts b/examples/reference-erc20/src/api/index.ts index 114868a35..e4c35118c 100644 --- a/examples/reference-erc20/src/api/index.ts +++ b/examples/reference-erc20/src/api/index.ts @@ -2,13 +2,11 @@ import { ponder } from "@/generated"; import { desc, graphql } from "@ponder/core"; import { formatEther } from "viem"; -// write file -ponder.use("/graphql", graphql()); +ponder.use("/graphql", graphql()).get("/big", async (c) => { + const { Account } = c.tables; -ponder.get("/big", async (c) => { - const db = c.get("db"); - - const account = await db + const account = await c.db + // ^? .select({ balance: Account.balance }) .from(Account) .orderBy(desc(Account.balance)) diff --git a/packages/core/src/drizzle/index.ts b/packages/core/src/drizzle/index.ts deleted file mode 100644 index 4fa6fc6c0..000000000 --- a/packages/core/src/drizzle/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type { DrizzleDb } from "./db.js"; -export { createDrizzleDb, convertToDrizzleTable } from "./runtime.js"; -export type { ConvertToDrizzleTable } from "./table.js"; diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index 8af966440..1e128c91b 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -1,6 +1,5 @@ -import type { DatabaseConfig } from "@/config/database.js"; import type { DatabaseService } from "@/database/service.js"; -import type { Table as PonderTable } from "@/schema/common.js"; +import type { Schema } from "@/schema/common.js"; import { isEnumColumn, isJSONColumn, @@ -10,10 +9,9 @@ import { isScalarColumn, } from "@/schema/utils.js"; import { createSqliteDatabase } from "@/utils/sqlite.js"; -import type { Table } from "drizzle-orm"; import { drizzle as drizzleSQLite } from "drizzle-orm/better-sqlite3"; import { drizzle as drizzlePg } from "drizzle-orm/node-postgres"; -import { pgSchema, pgTable } from "drizzle-orm/pg-core"; +import { pgTable } from "drizzle-orm/pg-core"; import { doublePrecision as PgDoublePrecision, integer as PgInteger, @@ -51,82 +49,104 @@ export const createDrizzleDb = (database: DatabaseService) => { } }; -export const convertToDrizzleTable = ( - tableName: string, - table: PonderTable, - databaseConfig: DatabaseConfig, -): Table => { - const columns = Object.entries(table).reduce<{ [columnName: string]: any }>( - (acc, [columnName, column]) => { +type SQLiteTable = Parameters[1]; +type PostgresTable = Parameters[1]; +type DrizzleTable = { [tableName: string]: any }; + +export const convertSchemaToDrizzle = ( + schema: Schema, + database: DatabaseService, +) => { + const drizzleTables: { [tableName: string]: DrizzleTable } = {}; + + for (const [tableName, table] of Object.entries(schema)) { + const drizzleColumns: DrizzleTable = {}; + + for (const [columnName, column] of Object.entries(table)) { if (isMaterialColumn(column)) { if (isJSONColumn(column)) { - acc[columnName] = convertJsonColumn(columnName, databaseConfig.kind); + drizzleColumns[columnName] = convertJsonColumn( + columnName, + database.kind, + ); } else if (isEnumColumn(column)) { - acc[columnName] = convertEnumColumn(columnName, databaseConfig.kind); + drizzleColumns[columnName] = convertEnumColumn( + columnName, + database.kind, + ); } else if (isScalarColumn(column) || isReferenceColumn(column)) { switch (column[" scalar"]) { case "string": - acc[columnName] = convertStringColumn( + drizzleColumns[columnName] = convertStringColumn( columnName, - databaseConfig.kind, + database.kind, ); break; case "int": - acc[columnName] = convertIntColumn( + drizzleColumns[columnName] = convertIntColumn( columnName, - databaseConfig.kind, + database.kind, ); break; case "boolean": - acc[columnName] = convertBooleanColumn( + drizzleColumns[columnName] = convertBooleanColumn( columnName, - databaseConfig.kind, + database.kind, ); break; case "float": - acc[columnName] = convertFloatColumn( + drizzleColumns[columnName] = convertFloatColumn( columnName, - databaseConfig.kind, + database.kind, ); break; case "hex": - acc[columnName] = convertHexColumn( + drizzleColumns[columnName] = convertHexColumn( columnName, - databaseConfig.kind, + database.kind, ); break; case "bigint": - acc[columnName] = convertBigintColumn( + drizzleColumns[columnName] = convertBigintColumn( columnName, - databaseConfig.kind, + database.kind, ); break; } // apply column constraints if (columnName === "id") { - acc[columnName] = acc[columnName]!.primaryKey(); + drizzleColumns[columnName] = + drizzleColumns[columnName]!.primaryKey(); } else if (isOptionalColumn(column) === false) { - acc[columnName] = acc[columnName]!.notNull(); + drizzleColumns[columnName] = drizzleColumns[columnName]!.notNull(); } } } - return acc; - }, - {}, - ); - - if (databaseConfig.kind === "postgres") { - if (databaseConfig.schema === "public") return pgTable(tableName, columns); - return pgSchema(databaseConfig.schema).table(tableName, columns); - } else { - return sqliteTable(tableName, columns); + } + + if (database.kind === "postgres") { + // if (database.schema === "public") { + return pgTable(tableName, drizzleColumns as PostgresTable); + // } + // return pgSchema(database.schema).table( + // tableName, + // drizzleColumns as PostgresTable, + // ); + } else { + drizzleTables[tableName] = sqliteTable( + tableName, + drizzleColumns as SQLiteTable, + ); + } } + + return drizzleTables; }; const convertStringColumn = ( diff --git a/packages/core/src/hono/context.ts b/packages/core/src/hono/context.ts index f1ba6e94a..818280965 100644 --- a/packages/core/src/hono/context.ts +++ b/packages/core/src/hono/context.ts @@ -1,11 +1,28 @@ import type { DrizzleDb } from "@/drizzle/db.js"; +import type { ConvertToDrizzleTable } from "@/drizzle/table.js"; +import type { ExtractTableNames, Schema } from "@/schema/common.js"; import type { Env, Context as HonoContext, Input } from "hono"; -export type Context = { +export type Context< + schema extends Schema = Schema, + path extends string = string, + input extends Input = {}, +> = { /** * ... */ db: DrizzleDb; + /** + * + */ + tables: { + [tableName in ExtractTableNames]: ConvertToDrizzleTable< + tableName, + // @ts-ignore + schema[tableName]["table"], + schema + >; + }; /** * Hono request object. * @@ -45,6 +62,7 @@ export type Context = { }; export type MiddlewareContext< + schema extends Schema = Schema, path extends string = string, input extends Input = {}, > = HonoContext & { @@ -52,4 +70,15 @@ export type MiddlewareContext< * ... */ db: DrizzleDb; + /** + * + */ + tables: { + [tableName in ExtractTableNames]: ConvertToDrizzleTable< + tableName, + // @ts-ignore + schema[tableName]["table"], + schema + >; + }; }; diff --git a/packages/core/src/hono/handler.ts b/packages/core/src/hono/handler.ts index da1571f8c..5bb389b3e 100644 --- a/packages/core/src/hono/handler.ts +++ b/packages/core/src/hono/handler.ts @@ -1,29 +1,35 @@ +import type { Schema } from "@/schema/common.js"; import type { PonderHono } from "@/types/hono.js"; import type { BlankInput, HandlerResponse, Input, Next } from "hono/types"; import type { Context, MiddlewareContext } from "./context.js"; export type Handler< + schema extends Schema = Schema, path extends string = any, input extends Input = BlankInput, response extends HandlerResponse = any, -> = (c: Context) => response; +> = (c: Context) => response; export type MiddlewareHandler< + schema extends Schema = Schema, path extends string = string, input extends Input = {}, -> = (c: MiddlewareContext, next: Next) => Promise; +> = ( + c: MiddlewareContext, + next: Next, +) => Promise; type BasePath = "/"; -export type HandlerInterface = { +export type HandlerInterface = { // app.get(handler) < path extends string = BasePath, input extends Input = BlankInput, response extends HandlerResponse = any, >( - handler: Handler, - ): PonderHono; + handler: Handler, + ): PonderHono; // app.get(handler x2) < @@ -32,8 +38,11 @@ export type HandlerInterface = { input2 extends Input = input, response extends HandlerResponse = any, >( - ...handlers: [Handler, Handler] - ): PonderHono; + ...handlers: [ + Handler, + Handler, + ] + ): PonderHono; // app.get(path, handler) < @@ -42,8 +51,8 @@ export type HandlerInterface = { input extends Input = BlankInput, >( path: path, - handler: Handler, - ): PonderHono; + handler: Handler, + ): PonderHono; // app.get(handler x 3) < @@ -54,11 +63,11 @@ export type HandlerInterface = { input3 extends Input = input & input2, >( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x2) < @@ -69,10 +78,10 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - MiddlewareHandler, - Handler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(handler x 4) < @@ -84,12 +93,12 @@ export type HandlerInterface = { input4 extends Input = input & input2 & input3, >( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x3) < @@ -101,11 +110,11 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(handler x 5) < @@ -118,13 +127,13 @@ export type HandlerInterface = { input5 extends Input = input & input2 & input3 & input4, >( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x4) < @@ -137,12 +146,12 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(handler x 6) < @@ -156,14 +165,14 @@ export type HandlerInterface = { input6 extends Input = input & input2 & input3 & input4 & input5, >( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x5) < @@ -177,13 +186,13 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(handler x 7) < @@ -198,15 +207,15 @@ export type HandlerInterface = { input7 extends Input = input & input2 & input3 & input4 & input5 & input6, >( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x6) < @@ -221,14 +230,14 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(handler x 8) < @@ -250,16 +259,16 @@ export type HandlerInterface = { input7, >( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x7) < @@ -275,15 +284,15 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(handler x 9) < @@ -313,17 +322,17 @@ export type HandlerInterface = { input8, >( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x8) < @@ -346,16 +355,16 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(handler x 10) < @@ -394,18 +403,18 @@ export type HandlerInterface = { input9, >( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x9) < @@ -436,17 +445,17 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x10) < @@ -486,18 +495,18 @@ export type HandlerInterface = { >( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - Handler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + Handler, ] - ): PonderHono; + ): PonderHono; // app.get(...handlers[]) < @@ -505,8 +514,8 @@ export type HandlerInterface = { input extends Input = BlankInput, response extends HandlerResponse = any, >( - ...handlers: Handler[] - ): PonderHono; + ...handlers: Handler[] + ): PonderHono; // app.get(path, ...handlers[]) < @@ -515,231 +524,237 @@ export type HandlerInterface = { response extends HandlerResponse = any, >( path: path, - ...handlers: Handler[] - ): PonderHono; + ...handlers: Handler[] + ): PonderHono; // app.get(path) - (path: path): PonderHono; + (path: path): PonderHono; }; -export interface MiddlewareHandlerInterface { +export interface MiddlewareHandlerInterface { //// app.use(...handlers[]) - (...handlers: MiddlewareHandler[]): PonderHono; + (...handlers: MiddlewareHandler[]): PonderHono; // app.use(handler) - (handler: MiddlewareHandler): PonderHono; + (handler: MiddlewareHandler): PonderHono; // app.use(handler x2) ( - ...handlers: [MiddlewareHandler, MiddlewareHandler] - ): PonderHono; + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; // app.get(path, handler) ( path: path, - handler: MiddlewareHandler, - ): PonderHono; + handler: MiddlewareHandler, + ): PonderHono; // app.use(handler x3) ( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x2) ( path: path, - ...handlers: [MiddlewareHandler, MiddlewareHandler] - ): PonderHono; + ...handlers: [ + MiddlewareHandler, + MiddlewareHandler, + ] + ): PonderHono; // app.use(handler x4) ( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x3) ( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.use(handler x5) ( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x4) ( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.use(handler x6) ( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x5) ( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.use(handler x7) ( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x6) ( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.use(handler x8) ( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x7) ( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.use(handler x9) ( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x8) ( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.use(handler x10) ( ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; // app.get(path, handler x9) ( path: path, ...handlers: [ - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, - MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, + MiddlewareHandler, ] - ): PonderHono; + ): PonderHono; //// app.use(path, ...handlers[]) ( path: path, - ...handlers: MiddlewareHandler[] - ): PonderHono; + ...handlers: MiddlewareHandler[] + ): PonderHono; } diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index a17397f04..41e5b86b7 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -1,7 +1,7 @@ import http from "node:http"; import type { Common } from "@/common/common.js"; import type { DatabaseService } from "@/database/service.js"; -import { createDrizzleDb } from "@/drizzle/runtime.js"; +import { convertSchemaToDrizzle, createDrizzleDb } from "@/drizzle/runtime.js"; import { type PonderRoutes, applyHonoRoutes } from "@/hono/index.js"; import type { ReadonlyStore } from "@/indexing-store/store.js"; import type { Schema } from "@/schema/common.js"; @@ -105,8 +105,6 @@ export async function createServer({ } }); - const db = createDrizzleDb(database); - const contextMiddleware = createMiddleware(async (c, next) => { c.set("readonlyStore", readonlyStore); c.set("schema", schema); @@ -136,9 +134,12 @@ export async function createServer({ .join(", ")}]`, }); + const db = createDrizzleDb(database); + const tables = convertSchemaToDrizzle(schema, database); + hono.route( "/", - applyHonoRoutes(userApp, userRoutes, { db }).onError((error, c) => + applyHonoRoutes(userApp, userRoutes, { db, tables }).onError((error, c) => onError(error, c, common), ), ); diff --git a/packages/core/src/types/hono.ts b/packages/core/src/types/hono.ts index 8a19a621c..35ab9f6be 100644 --- a/packages/core/src/types/hono.ts +++ b/packages/core/src/types/hono.ts @@ -2,11 +2,12 @@ import type { HandlerInterface, MiddlewareHandlerInterface, } from "@/hono/handler.js"; +import type { Schema } from "@/schema/common.js"; import type { Hono } from "hono"; -export type PonderHono = { - get: HandlerInterface; - post: HandlerInterface; - use: MiddlewareHandlerInterface; +export type PonderHono = { + get: HandlerInterface; + post: HandlerInterface; + use: MiddlewareHandlerInterface; hono: Hono; }; diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index 133bdaa29..01de4905c 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -247,5 +247,5 @@ export namespace Virtual { }, ) => Promise | void, ) => void; - } & PonderHono; + } & PonderHono; } From b9b5f65bfc117232b72ce4bdcd4fba4263949b80 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 10:49:56 -0400 Subject: [PATCH 077/122] cleanup drizzle database setup --- packages/core/src/bin/commands/serve.ts | 16 +- packages/core/src/bin/utils/run.ts | 20 +-- packages/core/src/drizzle/runtime.test.ts | 171 +++++++++------------- packages/core/src/drizzle/runtime.ts | 23 +-- packages/core/src/server/service.ts | 21 ++- 5 files changed, 112 insertions(+), 139 deletions(-) diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index 5b68e3f28..f16f467a5 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -5,7 +5,6 @@ import { MetricsService } from "@/common/metrics.js"; import { buildOptions } from "@/common/options.js"; import { buildPayload, createTelemetry } from "@/common/telemetry.js"; import { PostgresDatabaseService } from "@/database/postgres/service.js"; -import { getReadonlyStore } from "@/indexing-store/readonly.js"; import { createServer } from "@/server/service.js"; import type { CliOptions } from "../ponder.js"; import { setupShutdown } from "../utils/shutdown.js"; @@ -96,24 +95,13 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { isReadonly: true, }); - const readonlyStore = getReadonlyStore({ - encoding: "postgres", - schema, - // Note: `ponder serve` serves data from the `publishSchema`. - namespaceInfo: { - userNamespace: databaseConfig.publishSchema, - }, - db: database.readonlyDb, - common, - }); - const server = await createServer({ app, routes, common, - database, - readonlyStore, schema, + database, + dbNamespace: databaseConfig.publishSchema, }); server.setHealthy(); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 4cc58df28..42b0b7789 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -94,21 +94,13 @@ export async function run({ syncStore = new PostgresSyncStore({ db: database.syncDb, common }); } - const readonlyStore = getReadonlyStore({ - encoding: database.kind, - schema, - namespaceInfo, - db: database.readonlyDb, - common, - }); - const server = await createServer({ app: build.app, routes: build.routes, common, - database, - readonlyStore, schema, + database, + dbNamespace: namespaceInfo.userNamespace, }); // This can be a long-running operation, so it's best to do it after @@ -186,6 +178,14 @@ export async function run({ }, }); + const readonlyStore = getReadonlyStore({ + encoding: database.kind, + schema, + namespaceInfo, + db: database.indexingDb, + common, + }); + const historicalStore = getHistoricalStore({ encoding: database.kind, schema, diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index 1be6b67a3..5cb7b559b 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -3,21 +3,17 @@ import { setupDatabaseServices, setupIsolatedDatabase, } from "@/_test/setup.js"; -import type { DatabaseService } from "@/database/service.js"; +import type { Context } from "@/hono/context.js"; import type { HistoricalStore } from "@/indexing-store/store.js"; import { createSchema } from "@/schema/schema.js"; import { eq } from "drizzle-orm"; import { beforeEach, expect, test } from "vitest"; import type { DrizzleDb } from "./db.js"; -import { convertToDrizzleTable, createDrizzleDb } from "./runtime.js"; +import { convertSchemaToDrizzle, createDrizzleDb } from "./runtime.js"; beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); -const createDb = (database: DatabaseService) => { - return createDrizzleDb(database) as unknown as DrizzleDb; -}; - test("runtime select", async (context) => { const schema = createSchema((p) => ({ table: p.createTable({ @@ -25,25 +21,21 @@ test("runtime select", async (context) => { }), })); - const { database, cleanup, indexingStore } = await setupDatabaseServices( - context, - { schema }, - ); - - const db = createDb(database); + const { database, cleanup, indexingStore, namespaceInfo } = + await setupDatabaseServices(context, { schema }); await indexingStore.create({ tableName: "table", id: "kyle" }); await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); - const rows = await db - .select() - .from( - convertToDrizzleTable( - "table", - schema.table.table, - context.databaseConfig, - ), - ); + const db = createDrizzleDb(database) as unknown as DrizzleDb; + + const drizzleTables = convertSchemaToDrizzle( + schema, + database, + namespaceInfo.userNamespace, + ) as Context["tables"]; + + const rows = await db.select().from(drizzleTables.table); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: "kyle" }); @@ -58,25 +50,21 @@ test("select hex", async (context) => { }), })); - const { database, cleanup, indexingStore } = await setupDatabaseServices( - context, - { schema }, - ); - - const db = createDb(database); + const { database, cleanup, indexingStore, namespaceInfo } = + await setupDatabaseServices(context, { schema }); await indexingStore.create({ tableName: "table", id: "0x1" }); await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); - const rows = await db - .select() - .from( - convertToDrizzleTable( - "table", - schema.table.table, - context.databaseConfig, - ), - ); + const db = createDrizzleDb(database) as unknown as DrizzleDb; + + const drizzleTables = convertSchemaToDrizzle( + schema, + database, + namespaceInfo.userNamespace, + ) as Context["tables"]; + + const rows = await db.select().from(drizzleTables.table); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: "0x01" }); @@ -91,25 +79,21 @@ test("select bigint", async (context) => { }), })); - const { database, cleanup, indexingStore } = await setupDatabaseServices( - context, - { schema }, - ); - - const db = createDb(database); + const { database, cleanup, indexingStore, namespaceInfo } = + await setupDatabaseServices(context, { schema }); await indexingStore.create({ tableName: "table", id: 1n }); await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); - const rows = await db - .select() - .from( - convertToDrizzleTable( - "table", - schema.table.table, - context.databaseConfig, - ), - ); + const db = createDrizzleDb(database) as unknown as DrizzleDb; + + const drizzleTables = convertSchemaToDrizzle( + schema, + database, + namespaceInfo.userNamespace, + ) as Context["tables"]; + + const rows = await db.select().from(drizzleTables.table); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: 1n }); @@ -125,12 +109,8 @@ test("select json", async (context) => { }), })); - const { database, cleanup, indexingStore } = await setupDatabaseServices( - context, - { schema }, - ); - - const db = createDb(database); + const { database, cleanup, indexingStore, namespaceInfo } = + await setupDatabaseServices(context, { schema }); await indexingStore.create({ tableName: "table", @@ -143,15 +123,15 @@ test("select json", async (context) => { }); await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); - const rows = await db - .select() - .from( - convertToDrizzleTable( - "table", - schema.table.table, - context.databaseConfig, - ), - ); + const db = createDrizzleDb(database) as unknown as DrizzleDb; + + const drizzleTables = convertSchemaToDrizzle( + schema, + database, + namespaceInfo.userNamespace, + ) as Context["tables"]; + + const rows = await db.select().from(drizzleTables.table); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: "1", json: { prop: 52 } }); @@ -168,12 +148,8 @@ test("select enum", async (context) => { }), })); - const { database, cleanup, indexingStore } = await setupDatabaseServices( - context, - { schema }, - ); - - const db = createDb(database); + const { database, cleanup, indexingStore, namespaceInfo } = + await setupDatabaseServices(context, { schema }); await indexingStore.create({ tableName: "table", @@ -182,15 +158,15 @@ test("select enum", async (context) => { }); await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); - const rows = await db - .select() - .from( - convertToDrizzleTable( - "table", - schema.table.table, - context.databaseConfig, - ), - ); + const db = createDrizzleDb(database) as unknown as DrizzleDb; + + const drizzleTables = convertSchemaToDrizzle( + schema, + database, + namespaceInfo.userNamespace, + ) as Context["tables"]; + + const rows = await db.select().from(drizzleTables.table); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ id: "1", en: "hi" }); @@ -211,12 +187,8 @@ test("select with join", async (context) => { }), })); - const { database, cleanup, indexingStore } = await setupDatabaseServices( - context, - { schema }, - ); - - const db = createDb(database); + const { database, cleanup, indexingStore, namespaceInfo } = + await setupDatabaseServices(context, { schema }); await indexingStore.create({ tableName: "account", @@ -233,22 +205,21 @@ test("select with join", async (context) => { }); await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); - const account = convertToDrizzleTable( - "account", - schema.account.table, - context.databaseConfig, - ); - const nft = convertToDrizzleTable( - "nft", - schema.nft.table, - context.databaseConfig, - ); + const db = createDrizzleDb(database) as unknown as DrizzleDb; + + const drizzleTables = convertSchemaToDrizzle( + schema, + database, + namespaceInfo.userNamespace, + ) as Context["tables"]; const rows = await db .select() - .from(account) - // @ts-ignore - .fullJoin(nft, eq(account.id, nft.owner)); + .from(drizzleTables.account) + .fullJoin( + drizzleTables.nft, + eq(drizzleTables.account.id, drizzleTables.nft.owner), + ); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index 1e128c91b..b70290381 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -8,10 +8,11 @@ import { isReferenceColumn, isScalarColumn, } from "@/schema/utils.js"; +import { getTables } from "@/schema/utils.js"; import { createSqliteDatabase } from "@/utils/sqlite.js"; import { drizzle as drizzleSQLite } from "drizzle-orm/better-sqlite3"; import { drizzle as drizzlePg } from "drizzle-orm/node-postgres"; -import { pgTable } from "drizzle-orm/pg-core"; +import { pgSchema, pgTable } from "drizzle-orm/pg-core"; import { doublePrecision as PgDoublePrecision, integer as PgInteger, @@ -56,10 +57,11 @@ type DrizzleTable = { [tableName: string]: any }; export const convertSchemaToDrizzle = ( schema: Schema, database: DatabaseService, + dbNamespace: string, ) => { const drizzleTables: { [tableName: string]: DrizzleTable } = {}; - for (const [tableName, table] of Object.entries(schema)) { + for (const [tableName, { table }] of Object.entries(getTables(schema))) { const drizzleColumns: DrizzleTable = {}; for (const [columnName, column] of Object.entries(table)) { @@ -131,13 +133,16 @@ export const convertSchemaToDrizzle = ( } if (database.kind === "postgres") { - // if (database.schema === "public") { - return pgTable(tableName, drizzleColumns as PostgresTable); - // } - // return pgSchema(database.schema).table( - // tableName, - // drizzleColumns as PostgresTable, - // ); + if (dbNamespace === "public") { + drizzleTables[tableName] = pgTable( + tableName, + drizzleColumns as PostgresTable, + ); + } + drizzleTables[tableName] = pgSchema(dbNamespace).table( + tableName, + drizzleColumns as PostgresTable, + ); } else { drizzleTables[tableName] = sqliteTable( tableName, diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 41e5b86b7..8895afb23 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -3,7 +3,7 @@ import type { Common } from "@/common/common.js"; import type { DatabaseService } from "@/database/service.js"; import { convertSchemaToDrizzle, createDrizzleDb } from "@/drizzle/runtime.js"; import { type PonderRoutes, applyHonoRoutes } from "@/hono/index.js"; -import type { ReadonlyStore } from "@/indexing-store/store.js"; +import { getReadonlyStore } from "@/indexing-store/readonly.js"; import type { Schema } from "@/schema/common.js"; import { startClock } from "@/utils/timer.js"; import { serve } from "@hono/node-server"; @@ -24,16 +24,16 @@ export async function createServer({ app: userApp, routes: userRoutes, common, - database, schema, - readonlyStore, + database, + dbNamespace, }: { app?: Hono; routes?: PonderRoutes; common: Common; - database: DatabaseService; schema: Schema; - readonlyStore: ReadonlyStore; + database: DatabaseService; + dbNamespace: string; }): Promise { // Create hono app @@ -105,6 +105,14 @@ export async function createServer({ } }); + const readonlyStore = getReadonlyStore({ + encoding: database.kind, + schema, + namespaceInfo: { userNamespace: dbNamespace }, + db: database.readonlyDb, + common, + }); + const contextMiddleware = createMiddleware(async (c, next) => { c.set("readonlyStore", readonlyStore); c.set("schema", schema); @@ -135,8 +143,9 @@ export async function createServer({ }); const db = createDrizzleDb(database); - const tables = convertSchemaToDrizzle(schema, database); + const tables = convertSchemaToDrizzle(schema, database, dbNamespace); + // apply user routes to hono instance, registering a custom error handler hono.route( "/", applyHonoRoutes(userApp, userRoutes, { db, tables }).onError((error, c) => From 3897be3a51b4c8eb4bd5d8dcac48f76e73a38adb Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 11:01:13 -0400 Subject: [PATCH 078/122] cleanup --- packages/core/package.json | 6 ++---- packages/core/src/hono/handler.test-d.ts | 0 packages/core/tsup.config.ts | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 packages/core/src/hono/handler.test-d.ts diff --git a/packages/core/package.json b/packages/core/package.json index 7a34f949b..3073f515c 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -23,10 +23,8 @@ "types": "./dist/index.d.ts", "typings": "./dist/index.d.ts", "exports": { - ".": { - "import": "./dist/index.js", - "types": "./dist/index.d.ts" - } + "import": "./dist/index.js", + "types": "./dist/index.d.ts" }, "scripts": { "build": "tsup", diff --git a/packages/core/src/hono/handler.test-d.ts b/packages/core/src/hono/handler.test-d.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 4cf94887a..f261292a0 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ name: "@ponder/core", - entry: ["src/index.ts", "src/bin/ponder.ts", "src/drizzle/virtual.ts"], + entry: ["src/index.ts", "src/bin/ponder.ts"], outDir: "dist", format: ["esm"], sourcemap: true, From ed57d992056633ad38a6150f1dc38b40373d43dc Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 13:47:56 -0400 Subject: [PATCH 079/122] use virtual module in shim --- packages/core/package.json | 1 - packages/core/src/build/plugin.test.ts | 79 ------------------------ packages/core/src/build/plugin.ts | 43 ++++--------- packages/core/src/build/service.ts | 85 ++++++++++++++++++-------- packages/core/src/common/options.ts | 4 +- pnpm-lock.yaml | 4 +- 6 files changed, 75 insertions(+), 141 deletions(-) delete mode 100644 packages/core/src/build/plugin.test.ts diff --git a/packages/core/package.json b/packages/core/package.json index 7f039d72f..849ce961f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -66,7 +66,6 @@ "http-terminator": "^3.2.0", "ink": "^4.4.1", "kysely": "^0.26.3", - "magic-string": "^0.30.5", "p-queue": "^7.4.1", "pg": "^8.11.3", "pg-connection-string": "^2.6.2", diff --git a/packages/core/src/build/plugin.test.ts b/packages/core/src/build/plugin.test.ts deleted file mode 100644 index 82f62ca8b..000000000 --- a/packages/core/src/build/plugin.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { expect, test } from "vitest"; -import { ponderRegex, replaceStateless, shim } from "./plugin.js"; - -test("regex matches basic", () => { - const code = `import { ponder } from "@/generated";\n`; - - expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, shim); - expect(s.toString().includes(shim)).toBe(true); -}); - -test("regex matches multiline", () => { - const code = - 'import { ponder } from "@/generated";\n' + - 'ponder.on("PrimitiveManager:Swap", async ({ event, context }) => {\n'; - - expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, shim); - expect(s.toString().includes(shim)).toBe(true); -}); - -test("regex matches import including types before", () => { - const code = 'import { type Context, ponder } from "@/generated";\n'; - - expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, shim); - expect(s.toString().includes(shim)).toBe(true); -}); - -test("regex matches import includinga types after", () => { - const code = 'import { ponder, type Context } from "@/generated";\n'; - - expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, shim); - expect(s.toString().includes(shim)).toBe(true); -}); - -test("regex matches import including newlines", () => { - const code = - "import {\n" + "ponder,\n" + "type Context,\n" + '} from "@/generated";\n'; - - expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, shim); - expect(s.toString().includes(shim)).toBe(true); -}); - -test("regex matches no trailing semicolon", () => { - const code = `import { ponder } from "@/generated"`; - - expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, shim); - expect(s.toString().includes(shim)).toBe(true); -}); - -test("regex matches no trailing single quote import", () => { - const code = `import { ponder } from '@/generated'`; - - expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, shim); - expect(s.toString().includes(shim)).toBe(true); -}); - -test("regex matches no trailing newline", () => { - const code = `import { ponder } from "@/generated";ponder.on("PrimitiveManager:Swap", async ({ event, context }) => {`; - - expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, shim); - expect(s.toString().includes(shim)).toBe(true); -}); - -test("regex matches preceding import", () => { - const code = - `import {decodeEventLog} from "viem";\n` + - `import {ponder} from "@/generated";\n`; - - expect(ponderRegex.test(code)).toBe(true); - const s = replaceStateless(code, ponderRegex, shim); - expect(s.toString().includes(shim)).toBe(true); -}); diff --git a/packages/core/src/build/plugin.ts b/packages/core/src/build/plugin.ts index 353ac87f5..7088e4a02 100644 --- a/packages/core/src/build/plugin.ts +++ b/packages/core/src/build/plugin.ts @@ -1,57 +1,40 @@ -import MagicString from "magic-string"; import type { Plugin } from "vite"; -export const ponderRegex = - /^import\s+\{[^}]*\bponder\b[^}]*\}\s+from\s+["']@\/generated["'];?.*$/gm; +const virtualModule = `import { Hono } from "hono"; -export const shim = `import { Hono } from "hono"; - -let __ponderHono = { +const ponderHono = { routes: [], get(...maybePathOrHandlers) { - this.routes.push({method: "GET", pathOrHandlers: maybePathOrHandlers}); + this.routes.push({ method: "GET", pathOrHandlers: maybePathOrHandlers }); return this; }, post(...maybePathOrHandlers) { - this.routes.push({method: "POST", pathOrHandlers: maybePathOrHandlers}); + this.routes.push({ method: "POST", pathOrHandlers: maybePathOrHandlers }); return this; }, use(...maybePathOrHandlers) { - this.routes.push({method: "USE", pathOrHandlers: maybePathOrHandlers}); + this.routes.push({ method: "USE", pathOrHandlers: maybePathOrHandlers }); return this; - } -} + }, +}; -export let ponder = { +const ponder = { + ...ponderHono, hono: new Hono(), - ...__ponderHono, fns: [], on(name, fn) { this.fns.push({ name, fn }); }, }; -`; -export function replaceStateless(code: string, regex: RegExp, shim: string) { - const s = new MagicString(code); - // MagicString.replace calls regex.exec(), which increments `lastIndex` - // on a match. We have to set this back to zero to use the same regex - // multiple times. - regex.lastIndex = 0; - s.replace(regex, shim); - return s; -} +export { ponder }; +`; export const vitePluginPonder = (): Plugin => { return { name: "ponder", - transform: (code, id) => { - if (ponderRegex.test(code)) { - const s = replaceStateless(code, ponderRegex, shim); - const transformed = s.toString(); - const sourcemap = s.generateMap({ source: id }); - return { code: transformed, map: sourcemap }; - } + load: (id) => { + if (id === "@/generated") return virtualModule; return null; }, }; diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 5172dec34..4768786e2 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -33,6 +33,7 @@ export type Service = { // static common: Common; srcRegex: RegExp; + serverRegex: RegExp; // vite viteDevServer: ViteDevServer; @@ -86,6 +87,13 @@ export const create = async ({ .replace(escapeRegex, "\\$&"); const srcRegex = new RegExp(`^${escapedSrcDir}/.*\\.(ts|js)$`); + const escapedServerDir = common.options.serverDir + // If on Windows, use a POSIX path for this regex. + .replace(/\\/g, "/") + // Escape special characters in the path. + .replace(escapeRegex, "\\$&"); + const serverRegex = new RegExp(`^${escapedServerDir}/.*\\.(ts|js)$`); + const viteLogger = { warnedMessages: new Set(), loggedErrors: new WeakSet(), @@ -137,6 +145,7 @@ export const create = async ({ return { common, srcRegex, + serverRegex, viteDevServer, viteNodeServer, viteNodeRunner, @@ -236,13 +245,23 @@ export const start = async ( const hasSchemaUpdate = invalidated.includes( common.options.schemaFile.replace(/\\/g, "/"), ); - const hasSrcUpdate = invalidated.some((file) => - buildService.srcRegex.test(file), + const hasSrcUpdate = invalidated.some( + (file) => + buildService.srcRegex.test(file) && + !buildService.serverRegex.test(file), + ); + const hasServerUpdate = invalidated.some((file) => + buildService.serverRegex.test(file), ); // This branch could trigger if you change a `note.txt` file within `src/`. // Note: We could probably do a better job filtering out files in `isFileIgnored`. - if (!hasConfigUpdate && !hasSchemaUpdate && !hasSrcUpdate) { + if ( + !hasConfigUpdate && + !hasSchemaUpdate && + !hasSrcUpdate && + !hasServerUpdate + ) { return; } @@ -280,7 +299,7 @@ export const start = async ( rawBuild.indexingFunctions = result; } - if (hasSrcUpdate) { + if (hasServerUpdate) { const result = await executeServer(buildService); if (result.status === "error") { onBuild({ status: "error", error: result.error }); @@ -375,10 +394,10 @@ const executeIndexingFunctions = async ( } | { status: "error"; error: Error } > => { + // TODO(kyle) ignore server file const pattern = path .join(buildService.common.options.srcDir, "**/*.{js,mjs,ts,mts}") .replace(/\\/g, "/"); - // TODO(kyle) ignore server file const files = glob.sync(pattern); const executeResults = await Promise.all( @@ -388,8 +407,6 @@ const executeIndexingFunctions = async ( })), ); - const indexingFunctions: RawIndexingFunctions = []; - for (const executeResult of executeResults) { if (executeResult.status === "error") { buildService.common.logger.error({ @@ -403,8 +420,6 @@ const executeIndexingFunctions = async ( return executeResult; } - - indexingFunctions.push(...(executeResult.exports?.ponder?.fns ?? [])); } // Note that we are only hashing the file contents, not the exports. This is @@ -424,7 +439,13 @@ const executeIndexingFunctions = async ( } const contentHash = hash.digest("hex"); - return { status: "success", indexingFunctions, contentHash }; + const exports = await buildService.viteNodeRunner.executeId("@/generated"); + + return { + status: "success", + indexingFunctions: exports.ponder.fns, + contentHash, + }; }; const executeServer = async ( @@ -437,33 +458,45 @@ const executeServer = async ( } | { status: "error"; error: Error } > => { - const doesServerExist = fs.existsSync(buildService.common.options.serverFile); + const doesServerExist = fs.existsSync(buildService.common.options.serverDir); if (doesServerExist === false) { return { status: "success" }; } - const executeResult = await executeFile(buildService, { - file: buildService.common.options.serverFile, - }); + const pattern = path + .join(buildService.common.options.serverDir, "**/*.{js,mjs,ts,mts}") + .replace(/\\/g, "/"); + const files = glob.sync(pattern); - if (executeResult.status === "error") { - buildService.common.logger.error({ - service: "build", - msg: `Error while executing '${path.relative( - buildService.common.options.rootDir, - buildService.common.options.serverFile, - )}':`, - error: executeResult.error, - }); + const executeResults = await Promise.all( + files.map(async (file) => ({ + ...(await executeFile(buildService, { file })), + file, + })), + ); - return executeResult; + for (const executeResult of executeResults) { + if (executeResult.status === "error") { + buildService.common.logger.error({ + service: "build", + msg: `Error while executing '${path.relative( + buildService.common.options.rootDir, + executeResult.file, + )}':`, + error: executeResult.error, + }); + + return executeResult; + } } + const exports = await buildService.viteNodeRunner.executeId("@/generated"); + return { status: "success", - app: executeResult.exports?.ponder?.hono, - routes: executeResult.exports?.ponder?.routes, + app: exports.ponder.hono, + routes: exports.ponder.routes, }; }; diff --git a/packages/core/src/common/options.ts b/packages/core/src/common/options.ts index 5cd63d945..7e4db1c8f 100644 --- a/packages/core/src/common/options.ts +++ b/packages/core/src/common/options.ts @@ -8,9 +8,9 @@ export type Options = { configFile: string; schemaFile: string; - serverFile: string; rootDir: string; srcDir: string; + serverDir: string; generatedDir: string; ponderDir: string; logDir: string; @@ -79,8 +79,8 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => { rootDir, configFile: path.join(rootDir, cliOptions.config), schemaFile: path.join(rootDir, "ponder.schema.ts"), - serverFile: path.join(rootDir, "src", "api", "index.ts"), srcDir: path.join(rootDir, "src"), + serverDir: path.join(rootDir, "src", "api"), generatedDir: path.join(rootDir, "generated"), ponderDir: path.join(rootDir, ".ponder"), logDir: path.join(rootDir, ".ponder", "logs"), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9380e0cdf..8bc463503 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -632,9 +632,6 @@ importers: kysely: specifier: ^0.26.3 version: 0.26.3 - magic-string: - specifier: ^0.30.5 - version: 0.30.5 p-queue: specifier: ^7.4.1 version: 7.4.1 @@ -9995,6 +9992,7 @@ packages: engines: {node: '>=12'} dependencies: '@jridgewell/sourcemap-codec': 1.4.15 + dev: true /make-error@1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} From 518d608ac72e2e08428965331c722fe051b537fe Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 14:57:48 -0400 Subject: [PATCH 080/122] separate indexing and server build --- packages/core/src/bin/commands/dev.ts | 88 +++++++++++----- packages/core/src/bin/commands/serve.ts | 14 +-- packages/core/src/bin/commands/start.ts | 18 +++- packages/core/src/bin/utils/run.ts | 14 +-- packages/core/src/bin/utils/runServer.ts | 54 ++++++++++ packages/core/src/build/index.ts | 4 +- packages/core/src/build/service.ts | 125 +++++++++++++++++------ 7 files changed, 238 insertions(+), 79 deletions(-) create mode 100644 packages/core/src/bin/utils/runServer.ts diff --git a/packages/core/src/bin/commands/dev.ts b/packages/core/src/bin/commands/dev.ts index 46f0494d8..e910ed673 100644 --- a/packages/core/src/bin/commands/dev.ts +++ b/packages/core/src/bin/commands/dev.ts @@ -1,6 +1,7 @@ import { existsSync } from "node:fs"; import path from "node:path"; import { type BuildResult, createBuildService } from "@/build/index.js"; +import type { Build, BuildResultServer } from "@/build/service.js"; import { createLogger } from "@/common/logger.js"; import { MetricsService } from "@/common/metrics.js"; import { buildOptions } from "@/common/options.js"; @@ -9,6 +10,7 @@ import { UiService } from "@/ui/service.js"; import { createQueue } from "@ponder/common"; import type { CliOptions } from "../ponder.js"; import { run } from "../utils/run.js"; +import { runServer } from "../utils/runServer.js"; import { setupShutdown } from "../utils/shutdown.js"; export async function dev({ cliOptions }: { cliOptions: CliOptions }) { @@ -53,9 +55,11 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { const uiService = new UiService({ common }); let cleanupReloadable = () => Promise.resolve(); + let cleanupReloadableServer = () => Promise.resolve(); const cleanup = async () => { await cleanupReloadable(); + await cleanupReloadableServer(); await buildService.kill(); await telemetry.kill(); uiService.kill(); @@ -63,31 +67,55 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); + let cachedBuild: Build; + const buildQueue = createQueue({ initialStart: true, concurrency: 1, - worker: async (result: BuildResult) => { - await cleanupReloadable(); - - if (result.status === "success") { - uiService.reset(); - metrics.resetMetrics(); - - cleanupReloadable = await run({ - common, - build: result.build, - onFatalError: () => { - shutdown({ reason: "Received fatal error", code: 1 }); - }, - onReloadableError: (error) => { - buildQueue.clear(); - buildQueue.add({ status: "error", error }); - }, - }); + worker: async ( + result: + | ({ type: "indexing" } & BuildResult) + | ({ type: "server" } & BuildResultServer), + ) => { + if (result.type === "indexing") { + await cleanupReloadable(); + + if (result.status === "success") { + uiService.reset(); + metrics.resetMetrics(); + + cachedBuild = result.build; + + cleanupReloadable = await run({ + common, + build: result.build, + onFatalError: () => { + shutdown({ reason: "Received fatal error", code: 1 }); + }, + onReloadableError: (error) => { + buildQueue.clear(); + buildQueue.add({ type: "indexing", status: "error", error }); + }, + }); + } else { + // This handles build failures and indexing errors on hot reload. + uiService.setReloadableError(); + cleanupReloadable = () => Promise.resolve(); + } } else { - // This handles build failures and indexing errors on hot reload. - uiService.setReloadableError(); - cleanupReloadable = () => Promise.resolve(); + await cleanupReloadableServer(); + + if (result.status === "success") { + cleanupReloadableServer = await runServer({ + common, + build: cachedBuild, + buildServer: result.build, + }); + } else { + // This handles build failures and indexing errors on hot reload. + uiService.setReloadableError(); + cleanupReloadable = () => Promise.resolve(); + } } }, }); @@ -96,11 +124,21 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { watch: true, onBuild: (buildResult) => { buildQueue.clear(); - buildQueue.add(buildResult); + buildQueue.add({ type: "indexing", ...buildResult }); + }, + }); + const initialResultServer = await buildService.startServer({ + watch: true, + onBuild: (buildResult) => { + buildQueue.clear(); + buildQueue.add({ type: "server", ...buildResult }); }, }); - if (initialResult.status === "error") { + if ( + initialResult.status === "error" || + initialResultServer.status === "error" + ) { await shutdown({ reason: "Failed intial build", code: 1 }); return cleanup; } @@ -110,7 +148,9 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { properties: { cli_command: "dev", ...buildPayload(initialResult.build) }, }); - buildQueue.add(initialResult); + buildQueue + .add({ type: "indexing", ...initialResult }) + .then(() => buildQueue.add({ type: "server", ...initialResultServer })); return async () => { buildQueue.pause(); diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index c91fa0bbc..e3e9a6942 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -50,22 +50,22 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); - const initialResult = await buildService.start({ watch: false }); + const buildResult = await buildService.start({ watch: false }); + const buildResultServer = await buildService.startServer({ watch: false }); // Once we have the initial build, we can kill the build service. await buildService.kill(); - if (initialResult.status === "error") { + if (buildResult.status === "error" || buildResultServer.status === "error") { await shutdown({ reason: "Failed intial build", code: 1 }); return cleanup; } telemetry.record({ name: "lifecycle:session_start", - properties: { cli_command: "serve", ...buildPayload(initialResult.build) }, + properties: { cli_command: "serve", ...buildPayload(buildResult.build) }, }); - const { databaseConfig, optionsConfig, schema, app, routes } = - initialResult.build; + const { databaseConfig, optionsConfig, schema } = buildResult.build; common.options = { ...common.options, ...optionsConfig }; @@ -96,8 +96,8 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { }); const server = await createServer({ - app, - routes, + app: buildResultServer.build?.app, + routes: buildResultServer.build?.routes, common, schema, database, diff --git a/packages/core/src/bin/commands/start.ts b/packages/core/src/bin/commands/start.ts index 9b318fd22..6ad6eac14 100644 --- a/packages/core/src/bin/commands/start.ts +++ b/packages/core/src/bin/commands/start.ts @@ -6,6 +6,7 @@ import { buildOptions } from "@/common/options.js"; import { buildPayload, createTelemetry } from "@/common/telemetry.js"; import type { CliOptions } from "../ponder.js"; import { run } from "../utils/run.js"; +import { runServer } from "../utils/runServer.js"; import { setupShutdown } from "../utils/shutdown.js"; export async function start({ cliOptions }: { cliOptions: CliOptions }) { @@ -41,31 +42,34 @@ export async function start({ cliOptions }: { cliOptions: CliOptions }) { const buildService = await createBuildService({ common }); let cleanupReloadable = () => Promise.resolve(); + let cleanupReloadableServer = () => Promise.resolve(); const cleanup = async () => { await cleanupReloadable(); + await cleanupReloadableServer(); await telemetry.kill(); }; const shutdown = setupShutdown({ common, cleanup }); - const initialResult = await buildService.start({ watch: false }); + const buildResult = await buildService.start({ watch: false }); + const buildResultServer = await buildService.startServer({ watch: false }); // Once we have the initial build, we can kill the build service. await buildService.kill(); - if (initialResult.status === "error") { + if (buildResult.status === "error" || buildResultServer.status === "error") { await shutdown({ reason: "Failed intial build", code: 1 }); return cleanup; } telemetry.record({ name: "lifecycle:session_start", - properties: { cli_command: "start", ...buildPayload(initialResult.build) }, + properties: { cli_command: "start", ...buildPayload(buildResult.build) }, }); cleanupReloadable = await run({ common, - build: initialResult.build, + build: buildResult.build, onFatalError: () => { shutdown({ reason: "Received fatal error", code: 1 }); }, @@ -74,5 +78,11 @@ export async function start({ cliOptions }: { cliOptions: CliOptions }) { }, }); + cleanupReloadableServer = await runServer({ + common, + build: buildResult.build, + buildServer: buildResultServer.build, + }); + return cleanup; } diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 6be1e473a..8572aa7bf 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -10,7 +10,6 @@ import { getReadonlyStore } from "@/indexing-store/readonly.js"; import { getRealtimeStore } from "@/indexing-store/realtime.js"; import type { IndexingStore, Status } from "@/indexing-store/store.js"; import { createIndexingService } from "@/indexing/index.js"; -import { createServer } from "@/server/service.js"; import { PostgresSyncStore } from "@/sync-store/postgres/store.js"; import { SqliteSyncStore } from "@/sync-store/sqlite/store.js"; import type { SyncStore } from "@/sync-store/store.js"; @@ -41,7 +40,7 @@ export type RealtimeEvent = }; /** - * Starts the server, sync, and indexing services for the specified build. + * Starts the sync and indexing services for the specified build. */ export async function run({ common, @@ -104,15 +103,6 @@ export async function run({ syncStore = new PostgresSyncStore({ db: database.syncDb, common }); } - const server = await createServer({ - app: build.app, - routes: build.routes, - common, - schema, - database, - dbNamespace: namespaceInfo.userNamespace, - }); - const metadataStore = getMetadataStore({ encoding: database.kind, namespaceInfo, @@ -347,14 +337,12 @@ export async function run({ const startPromise = start(); return async () => { - const serverPromise = server.kill(); indexingService.kill(); await syncService.kill(); realtimeQueue.pause(); realtimeQueue.clear(); await realtimeQueue.onIdle(); await startPromise; - await serverPromise; await database.kill(); }; } diff --git a/packages/core/src/bin/utils/runServer.ts b/packages/core/src/bin/utils/runServer.ts new file mode 100644 index 000000000..7cffbdaff --- /dev/null +++ b/packages/core/src/bin/utils/runServer.ts @@ -0,0 +1,54 @@ +import type { Build } from "@/build/index.js"; +import type { BuildServer } from "@/build/service.js"; +import type { Common } from "@/common/common.js"; +import { PostgresDatabaseService } from "@/database/postgres/service.js"; +import type { DatabaseService } from "@/database/service.js"; +import { SqliteDatabaseService } from "@/database/sqlite/service.js"; +import { createServer } from "@/server/service.js"; + +/** + * Starts the server for the specified build. + */ +export async function runServer({ + common, + build, + buildServer, +}: { + common: Common; + build: Build; + buildServer: BuildServer | undefined; +}) { + const { databaseConfig, optionsConfig, schema } = build; + + common.options = { ...common.options, ...optionsConfig }; + + let database: DatabaseService; + + if (databaseConfig.kind === "sqlite") { + const { directory } = databaseConfig; + database = new SqliteDatabaseService({ common, directory }); + } else { + const { poolConfig, schema: userNamespace, publishSchema } = databaseConfig; + database = new PostgresDatabaseService({ + common, + poolConfig, + userNamespace, + publishSchema, + }); + } + + const server = await createServer({ + app: buildServer?.app, + routes: buildServer?.routes, + common, + schema, + database, + dbNamespace: + databaseConfig.kind === "sqlite" ? "public" : databaseConfig.schema, + }); + + return async () => { + await server.kill(); + await database.kill(); + }; +} diff --git a/packages/core/src/build/index.ts b/packages/core/src/build/index.ts index 366a47b05..c36d6b6c8 100644 --- a/packages/core/src/build/index.ts +++ b/packages/core/src/build/index.ts @@ -1,8 +1,8 @@ import { type Extend, extend } from "@/utils/extend.js"; -import { create, kill, start } from "./service.js"; +import { create, kill, start, startServer } from "./service.js"; import type { Build, BuildResult, Service } from "./service.js"; -const methods = { start, kill }; +const methods = { start, startServer, kill }; export const createBuildService = extend(create, methods); diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 4768786e2..8d0cc6739 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -54,15 +54,21 @@ export type Build = { graphQLSchema: GraphQLSchema; // Indexing functions indexingFunctions: IndexingFunctions; - // Server - app?: Hono; - routes?: PonderRoutes; +}; + +export type BuildServer = { + app: Hono; + routes: PonderRoutes; }; export type BuildResult = | { status: "success"; build: Build } | { status: "error"; error: Error }; +export type BuildResultServer = + | { status: "success"; build?: BuildServer } + | { status: "error"; error: Error }; + type RawBuild = { config: { config: Config; contentHash: string }; schema: { schema: Schema; contentHash: string }; @@ -70,7 +76,6 @@ type RawBuild = { indexingFunctions: RawIndexingFunctions; contentHash: string; }; - server: { app?: Hono; routes?: PonderRoutes }; }; export const create = async ({ @@ -174,7 +179,6 @@ export const start = async ( const configResult = await executeConfig(buildService); const schemaResult = await executeSchema(buildService); const indexingFunctionsResult = await executeIndexingFunctions(buildService); - const serverResult = await executeServer(buildService); if (configResult.status === "error") { return { status: "error", error: configResult.error }; @@ -185,15 +189,11 @@ export const start = async ( if (indexingFunctionsResult.status === "error") { return { status: "error", error: indexingFunctionsResult.error }; } - if (serverResult.status === "error") { - return { status: "error", error: serverResult.error }; - } const rawBuild: RawBuild = { config: configResult, schema: schemaResult, indexingFunctions: indexingFunctionsResult, - server: serverResult, }; const buildResult = await validateAndBuild(buildService, rawBuild); @@ -250,18 +250,10 @@ export const start = async ( buildService.srcRegex.test(file) && !buildService.serverRegex.test(file), ); - const hasServerUpdate = invalidated.some((file) => - buildService.serverRegex.test(file), - ); // This branch could trigger if you change a `note.txt` file within `src/`. // Note: We could probably do a better job filtering out files in `isFileIgnored`. - if ( - !hasConfigUpdate && - !hasSchemaUpdate && - !hasSrcUpdate && - !hasServerUpdate - ) { + if (!hasConfigUpdate && !hasSchemaUpdate && !hasSrcUpdate) { return; } @@ -299,15 +291,6 @@ export const start = async ( rawBuild.indexingFunctions = result; } - if (hasServerUpdate) { - const result = await executeServer(buildService); - if (result.status === "error") { - onBuild({ status: "error", error: result.error }); - return; - } - rawBuild.server = result; - } - const buildResult = await validateAndBuild(buildService, rawBuild); onBuild(buildResult); }; @@ -318,6 +301,92 @@ export const start = async ( return buildResult; }; +export const startServer = async ( + buildService: Service, + { + watch, + onBuild, + }: + | { watch: true; onBuild: (buildResult: BuildResultServer) => void } + | { watch: false; onBuild?: never }, +): Promise => { + const { common } = buildService; + + const buildResult = await executeServer(buildService); + + if (buildResult.status === "error") { + return { status: "error", error: buildResult.error }; + } + + // If watch is false (`ponder start` or `ponder serve`), + // don't register any event handlers on the watcher. + if (watch) { + // Define the directories and files to ignore + const ignoredDirs = [common.options.generatedDir, common.options.ponderDir]; + const ignoredFiles = [ + path.join(common.options.rootDir, "ponder-env.d.ts"), + path.join(common.options.rootDir, ".env.local"), + ]; + + const isFileIgnored = (filePath: string) => { + const isInIgnoredDir = ignoredDirs.some((dir) => { + const rel = path.relative(dir, filePath); + return !rel.startsWith("..") && !path.isAbsolute(rel); + }); + + const isIgnoredFile = ignoredFiles.includes(filePath); + return isInIgnoredDir || isIgnoredFile; + }; + + const onFileChange = async (_file: string) => { + if (isFileIgnored(_file)) return; + + // Note that `toFilePath` always returns a POSIX path, even if you pass a Windows path. + const file = toFilePath( + normalizeModuleId(_file), + common.options.rootDir, + ).path; + + // Invalidate all modules that depend on the updated files. + // Note that `invalidateDepTree` accepts and returns POSIX paths, even on Windows. + const invalidated = [ + ...buildService.viteNodeRunner.moduleCache.invalidateDepTree([file]), + ]; + + // If no files were invalidated, no need to reload. + if (invalidated.length === 0) return; + + const hasServerUpdate = invalidated.some((file) => + buildService.serverRegex.test(file), + ); + + // This branch could trigger if you change a `note.txt` file within `src/`. + // Note: We could probably do a better job filtering out files in `isFileIgnored`. + if (!hasServerUpdate) return; + + common.logger.info({ + service: "build", + msg: `Hot reload ${invalidated + .map((f) => `'${path.relative(common.options.rootDir, f)}'`) + .join(", ")}`, + }); + + const result = await executeServer(buildService); + if (result.status === "error") { + onBuild({ status: "error", error: result.error }); + return; + } + }; + + buildService.viteDevServer.watcher.on("change", onFileChange); + } + + return { + status: "success", + build: buildResult.app ? (buildResult as BuildServer) : undefined, + }; +}; + export const kill = async (buildService: Service): Promise => { await buildService.viteDevServer?.close(); buildService.common.logger.debug({ @@ -570,8 +639,6 @@ const validateAndBuild = async ( graphQLSchema, indexingFunctions: buildConfigAndIndexingFunctionsResult.indexingFunctions, - app: rawBuild.server.app, - routes: rawBuild.server.routes, }, }; }; From 08235b4f6b01fd0929e8954d260d5c7e7c69c896 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 15:53:40 -0400 Subject: [PATCH 081/122] change name of metadata table and metadata graphql field, update shape of status --- packages/core/src/bin/utils/run.ts | 24 ++++--------- .../src/database/postgres/service.test.ts | 34 +++++++++---------- .../core/src/database/postgres/service.ts | 6 ++-- .../core/src/database/sqlite/service.test.ts | 14 ++++---- packages/core/src/database/sqlite/service.ts | 4 +-- .../core/src/indexing-store/metadata.test.ts | 4 +-- packages/core/src/indexing-store/metadata.ts | 8 ++--- packages/core/src/indexing-store/store.ts | 3 +- .../server/graphql/buildGraphqlSchema.test.ts | 16 +++++---- .../src/server/graphql/buildGraphqlSchema.ts | 2 +- packages/core/src/server/graphql/metadata.ts | 2 +- packages/core/src/sync/index.ts | 4 +-- packages/core/src/sync/service.ts | 15 ++++---- 13 files changed, 63 insertions(+), 73 deletions(-) diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 2a5c168f6..bada6d461 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -76,8 +76,7 @@ export async function run({ for (const network of networks) { status[network.name] = { ready: false, - blockNumber: null, - blockTimestamp: null, + block: null, }; } @@ -186,19 +185,11 @@ export async function run({ // set status to most recently processed realtime block or end block // for each chain. - const checkpointStatus = syncService.getRealtimeStatus(); + const statusBlocks = syncService.getStatusBlocks(); for (const network of networks) { - status[network.name] = { - ready: true, - blockNumber: - checkpointStatus[network.name]?.blockNumber ?? - status[network.name]?.blockNumber ?? - null, - blockTimestamp: - checkpointStatus[network.name]?.blockTimestamp ?? - status[network.name]?.blockTimestamp ?? - null, - }; + if (statusBlocks[network.name] !== undefined) { + status[network.name]!.block = statusBlocks[network.name]!; + } } await metadataStore.setStatus(status); @@ -326,12 +317,11 @@ export async function run({ // set status to ready and set blocks to most recently processed // or end block - const checkpointStatus = syncService.getRealtimeStatus(); + const statusBlocks = syncService.getStatusBlocks(); for (const network of networks) { status[network.name] = { ready: true, - blockNumber: checkpointStatus[network.name]?.blockNumber ?? null, - blockTimestamp: checkpointStatus[network.name]?.blockTimestamp ?? null, + block: statusBlocks[network.name] ?? null, }; } diff --git a/packages/core/src/database/postgres/service.test.ts b/packages/core/src/database/postgres/service.test.ts index 597a3a138..ab282a5f7 100644 --- a/packages/core/src/database/postgres/service.test.ts +++ b/packages/core/src/database/postgres/service.test.ts @@ -79,7 +79,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { ]); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", ]); @@ -114,7 +114,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", ]); @@ -129,7 +129,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Dog", "Apple", ]); @@ -161,14 +161,14 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", ]); expect(await getViewNames(databaseTwo.db, "publish")).toStrictEqual([ "Pet", "Person", - "ponder_metadata", + "_ponder_metadata", ]); await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); @@ -182,14 +182,14 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Dog", "Apple", ]); expect(await getViewNames(databaseTwo.db, "publish")).toStrictEqual([ "Dog", "Apple", - "ponder_metadata", + "_ponder_metadata", ]); await databaseTwo.kill(); @@ -222,7 +222,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { ); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", "not_a_ponder_table", @@ -232,7 +232,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "not_a_ponder_table", "AnotherTable", "Dog", @@ -571,7 +571,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public2", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Dog", "Apple", ]); @@ -616,7 +616,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public2", "abc", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Dog", "Apple", ]); @@ -637,7 +637,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", ]); @@ -647,7 +647,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getViewNames(database.db, "publish")).toStrictEqual([ "Pet", "Person", - "ponder_metadata", + "_ponder_metadata", ]); await database.kill(); @@ -665,7 +665,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", ]); @@ -680,7 +680,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getTableNames(database.db, "publish")).toStrictEqual(["Pet"]); expect(await getViewNames(database.db, "publish")).toStrictEqual([ "Person", - "ponder_metadata", + "_ponder_metadata", ]); await database.kill(); @@ -698,7 +698,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", ]); @@ -718,7 +718,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getViewNames(database.db, "nice_looks-great")).toStrictEqual([ "Pet", "Person", - "ponder_metadata", + "_ponder_metadata", ]); await database.kill(); diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index 88bafe169..ad3721bf8 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -265,7 +265,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { // Create ponder_metadata table if it doesn't exist await tx.schema .withSchema(this.userNamespace) - .createTable("ponder_metadata") + .createTable("_ponder_metadata") .addColumn("key", "text", (col) => col.primaryKey()) .addColumn("value", "jsonb") .ifNotExists() @@ -275,7 +275,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { await tx .withSchema(this.userNamespace) // @ts-expect-error Kysely doesn't have types for user schema - .insertInto("ponder_metadata") + .insertInto("_ponder_metadata") // @ts-expect-error Kysely doesn't have types for user schema .values({ key: "status", value: null }) // @ts-expect-error Kysely doesn't have types for user schema @@ -589,7 +589,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { await tx.schema.createSchema(publishSchema).ifNotExists().execute(); for (const tableName of Object.keys(getTables(this.schema)).concat( - "ponder_metadata", + "_ponder_metadata", )) { // Check if there is an existing relation with the name we're about to publish. const result = await tx.executeQuery<{ diff --git a/packages/core/src/database/sqlite/service.test.ts b/packages/core/src/database/sqlite/service.test.ts index 0d9a65901..c8b686732 100644 --- a/packages/core/src/database/sqlite/service.test.ts +++ b/packages/core/src/database/sqlite/service.test.ts @@ -78,7 +78,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { ]); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", ]); @@ -111,7 +111,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", ]); @@ -126,7 +126,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Dog", "Apple", ]); @@ -159,7 +159,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { ); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Pet", "Person", "not_a_ponder_table", @@ -169,7 +169,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "not_a_ponder_table", "AnotherTable", "Dog", @@ -494,7 +494,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public2", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Dog", "Apple", ]); @@ -538,7 +538,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public2", "abc", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "ponder_metadata", + "_ponder_metadata", "Dog", "Apple", ]); diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index 5b767d215..e7c036693 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -236,7 +236,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { // Create ponder_metadata table if it doesn't exist await tx.schema .withSchema(this.userNamespace) - .createTable("ponder_metadata") + .createTable("_ponder_metadata") .addColumn("key", "text", (col) => col.primaryKey()) .addColumn("value", "jsonb") .ifNotExists() @@ -246,7 +246,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { await tx .withSchema(this.userNamespace) // @ts-expect-error Kysely doesn't have types for user schema - .insertInto("ponder_metadata") + .insertInto("_ponder_metadata") // @ts-expect-error Kysely doesn't have types for user schema .values({ key: "status", value: null }) // @ts-expect-error Kysely doesn't have types for user schema diff --git a/packages/core/src/indexing-store/metadata.test.ts b/packages/core/src/indexing-store/metadata.test.ts index 1d8aa065f..648722425 100644 --- a/packages/core/src/indexing-store/metadata.test.ts +++ b/packages/core/src/indexing-store/metadata.test.ts @@ -46,13 +46,13 @@ test("setMetadata()", async (context) => { }); await metadataStore.setStatus({ - mainnet: { blockNumber: 10, blockTimestamp: 10, ready: false }, + mainnet: { block: { number: 10, timestamp: 10 }, ready: false }, }); const status = await metadataStore.getStatus(); expect(status).toStrictEqual({ - mainnet: { blockNumber: 10, blockTimestamp: 10, ready: false }, + mainnet: { block: { number: 10, timestamp: 10 }, ready: false }, }); await cleanup(); diff --git a/packages/core/src/indexing-store/metadata.ts b/packages/core/src/indexing-store/metadata.ts index b4b9bf1ed..c59701b2d 100644 --- a/packages/core/src/indexing-store/metadata.ts +++ b/packages/core/src/indexing-store/metadata.ts @@ -12,10 +12,10 @@ export const getMetadataStore = ({ db: HeadlessKysely; }): MetadataStore => ({ getStatus: async () => { - return db.wrap({ method: "_metadata.getLatest()" }, async () => { + return db.wrap({ method: "_ponder_metadata.getLatest()" }, async () => { const metadata = await db .withSchema(namespaceInfo.userNamespace) - .selectFrom("ponder_metadata") + .selectFrom("_ponder_metadata") .select("value") .where("key", "=", "status") .executeTakeFirst(); @@ -28,10 +28,10 @@ export const getMetadataStore = ({ }); }, setStatus: (status: Status) => { - return db.wrap({ method: "_metadata.setLatest()" }, async () => { + return db.wrap({ method: "_ponder_metadata.setLatest()" }, async () => { await db .withSchema(namespaceInfo.userNamespace) - .insertInto("ponder_metadata") + .insertInto("_ponder_metadata") .values({ key: "status", value: encoding === "sqlite" ? JSON.stringify(status) : status, diff --git a/packages/core/src/indexing-store/store.ts b/packages/core/src/indexing-store/store.ts index 35128a788..61adf9df6 100644 --- a/packages/core/src/indexing-store/store.ts +++ b/packages/core/src/indexing-store/store.ts @@ -101,8 +101,7 @@ export type HistoricalStore = ReadonlyStore & export type Status = { [networkName: string]: { - blockNumber: number | null; - blockTimestamp: number | null; + block: { number: number; timestamp: number } | null; ready: boolean; }; }; diff --git a/packages/core/src/server/graphql/buildGraphqlSchema.test.ts b/packages/core/src/server/graphql/buildGraphqlSchema.test.ts index b9828a053..51ae22ca0 100644 --- a/packages/core/src/server/graphql/buildGraphqlSchema.test.ts +++ b/packages/core/src/server/graphql/buildGraphqlSchema.test.ts @@ -3055,8 +3055,10 @@ test("metadata", async (context) => { await metadataStore.setStatus({ mainnet: { ready: true, - blockNumber: 10, - blockTimestamp: 20, + block: { + number: 10, + timestamp: 20, + }, }, }); @@ -3064,7 +3066,7 @@ test("metadata", async (context) => { const document = parse(` query { - _metadata { + _meta { status } } @@ -3077,12 +3079,14 @@ test("metadata", async (context) => { }); expect(result.data).toMatchObject({ - _metadata: { + _meta: { status: { mainnet: { ready: true, - blockNumber: 10, - blockTimestamp: 20, + block: { + number: 10, + timestamp: 20, + }, }, }, }, diff --git a/packages/core/src/server/graphql/buildGraphqlSchema.ts b/packages/core/src/server/graphql/buildGraphqlSchema.ts index 8bdee6443..87978df62 100644 --- a/packages/core/src/server/graphql/buildGraphqlSchema.ts +++ b/packages/core/src/server/graphql/buildGraphqlSchema.ts @@ -54,7 +54,7 @@ export const buildGraphqlSchema = (schema: Schema): GraphQLSchema => { }); } - queryFields._metadata = { + queryFields._meta = { type: metadataEntity, resolve: async (_source, _args, context) => { const status = await context.metadataStore.getStatus(); diff --git a/packages/core/src/server/graphql/metadata.ts b/packages/core/src/server/graphql/metadata.ts index 933f9705c..c6ba3e8d2 100644 --- a/packages/core/src/server/graphql/metadata.ts +++ b/packages/core/src/server/graphql/metadata.ts @@ -2,6 +2,6 @@ import { GraphQLObjectType } from "graphql"; import { GraphQLJSON } from "graphql-type-json"; export const metadataEntity = new GraphQLObjectType({ - name: "_metadata", + name: "_meta", fields: { status: { type: GraphQLJSON } }, }); diff --git a/packages/core/src/sync/index.ts b/packages/core/src/sync/index.ts index 5de371440..2e7b46bdf 100644 --- a/packages/core/src/sync/index.ts +++ b/packages/core/src/sync/index.ts @@ -22,7 +22,7 @@ import { create, getCachedTransport, getHistoricalCheckpoint, - getRealtimeStatus, + getStatusBlocks, kill, startHistorical, startRealtime, @@ -31,7 +31,7 @@ import { const methods = { startHistorical, getHistoricalCheckpoint, - getRealtimeStatus, + getStatusBlocks, startRealtime, getCachedTransport, kill, diff --git a/packages/core/src/sync/service.ts b/packages/core/src/sync/service.ts index 699b6d1e6..f20eafc12 100644 --- a/packages/core/src/sync/service.ts +++ b/packages/core/src/sync/service.ts @@ -567,18 +567,16 @@ export const kill = async (syncService: Service) => { }; /** Return the number and timestamp of the most recently processed blocks. */ -export const getRealtimeStatus = (syncService: Service) => { +export const getStatusBlocks = (syncService: Service) => { const status: { - [networkName: string]: - | { blockNumber: number; blockTimestamp: number } - | undefined; + [networkName: string]: { number: number; timestamp: number } | undefined; } = {}; for (const networkService of syncService.networkServices) { if (networkService.realtime === undefined) { status[networkService.network.name] = { - blockNumber: Number(networkService.endCheckpoint!.blockNumber), - blockTimestamp: networkService.endCheckpoint!.blockTimestamp, + number: Number(networkService.endCheckpoint!.blockNumber), + timestamp: networkService.endCheckpoint!.blockTimestamp, }; } else { const mostRecentBlock = @@ -590,9 +588,8 @@ export const getRealtimeStatus = (syncService: Service) => { status[networkService.network.name] = undefined; } else { status[networkService.network.name] = { - ...zeroCheckpoint, - blockTimestamp: mostRecentBlock.timestamp, - blockNumber: mostRecentBlock.number, + timestamp: mostRecentBlock.timestamp, + number: mostRecentBlock.number, }; } } From 95f9313645d1fc7a83a31bce4cdf87b42e9f4c35 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 16:17:02 -0400 Subject: [PATCH 082/122] use status in e2e tests --- .../core/src/_test/e2e/erc20/erc20.test.ts | 40 +++++-------------- .../src/_test/e2e/factory/factory.test.ts | 15 ++++--- packages/core/src/_test/utils.ts | 22 +++++++--- packages/core/src/bin/utils/run.ts | 6 ++- 4 files changed, 38 insertions(+), 45 deletions(-) diff --git a/packages/core/src/_test/e2e/erc20/erc20.test.ts b/packages/core/src/_test/e2e/erc20/erc20.test.ts index bae6e8801..62c244e11 100644 --- a/packages/core/src/_test/e2e/erc20/erc20.test.ts +++ b/packages/core/src/_test/e2e/erc20/erc20.test.ts @@ -6,11 +6,14 @@ import { setupIsolatedDatabase, } from "@/_test/setup.js"; import { simulate } from "@/_test/simulate.js"; -import { getFreePort, postGraphql, waitForHealthy } from "@/_test/utils.js"; +import { + getFreePort, + postGraphql, + waitForIndexedBlock, +} from "@/_test/utils.js"; import { serve } from "@/bin/commands/serve.js"; import { start } from "@/bin/commands/start.js"; import { range } from "@/utils/range.js"; -import { wait } from "@/utils/wait.js"; import { rimrafSync } from "rimraf"; import { zeroAddress } from "viem"; import { beforeEach, describe, expect, test } from "vitest"; @@ -43,35 +46,14 @@ test("erc20", async (context) => { }, }); - await waitForHealthy(port); - - let response = await postGraphql( - port, - ` - accounts { - items { - id - balance - } - } - `, - ); - - expect(response.status).toBe(200); - let body = (await response.json()) as any; - expect(body.errors).toBe(undefined); - let accounts = body.data.accounts.items; - expect(accounts).toHaveLength(0); - await simulate({ erc20Address: context.erc20.address, factoryAddress: context.factory.address, }); - // TODO: Find a consistent way to wait for indexing to be complete. - await wait(2500); + await waitForIndexedBlock(port, "mainnet", 8); - response = await postGraphql( + const response = await postGraphql( port, ` accounts { @@ -84,9 +66,9 @@ test("erc20", async (context) => { ); expect(response.status).toBe(200); - body = (await response.json()) as any; + const body = (await response.json()) as any; expect(body.errors).toBe(undefined); - accounts = body.data.accounts.items; + const accounts = body.data.accounts.items; expect(accounts[0]).toMatchObject({ id: zeroAddress, @@ -119,8 +101,6 @@ describe.skipIf(shouldSkip)("postgres database", () => { }, }); - await waitForHealthy(startPort); - for (const _ in range(0, 3)) { await simulate({ erc20Address: context.erc20.address, @@ -138,8 +118,6 @@ describe.skipIf(shouldSkip)("postgres database", () => { }, }); - await waitForHealthy(servePort); - const response = await postGraphql( servePort, ` diff --git a/packages/core/src/_test/e2e/factory/factory.test.ts b/packages/core/src/_test/e2e/factory/factory.test.ts index 516c04856..3ac29b09b 100644 --- a/packages/core/src/_test/e2e/factory/factory.test.ts +++ b/packages/core/src/_test/e2e/factory/factory.test.ts @@ -6,9 +6,12 @@ import { setupIsolatedDatabase, } from "@/_test/setup.js"; import { simulatePairSwap } from "@/_test/simulate.js"; -import { getFreePort, postGraphql, waitForHealthy } from "@/_test/utils.js"; +import { + getFreePort, + postGraphql, + waitForIndexedBlock, +} from "@/_test/utils.js"; import { start } from "@/bin/commands/start.js"; -import { wait } from "@/utils/wait.js"; import { rimrafSync } from "rimraf"; import { beforeEach, expect, test } from "vitest"; @@ -40,10 +43,7 @@ test("factory", async (context) => { }, }); - await waitForHealthy(port); - - // TODO: Find a consistent way to wait for indexing to be complete. - await wait(500); + await waitForIndexedBlock(port, "mainnet", 4); let response = await postGraphql( port, @@ -74,8 +74,7 @@ test("factory", async (context) => { await simulatePairSwap(context.factory.pair); - // TODO: Find a consistent way to wait for indexing to be complete. - await wait(2500); + await waitForIndexedBlock(port, "mainnet", 5); response = await postGraphql( port, diff --git a/packages/core/src/_test/utils.ts b/packages/core/src/_test/utils.ts index 732a8ed1c..f9fbb3cba 100644 --- a/packages/core/src/_test/utils.ts +++ b/packages/core/src/_test/utils.ts @@ -9,6 +9,7 @@ import { sourceIsFactoryLog, sourceIsLog, } from "@/config/sources.js"; +import type { Status } from "@/indexing-store/store.js"; import type { RawEvent } from "@/sync-store/store.js"; import type { SyncBlock, @@ -682,18 +683,29 @@ export function getFreePort(): Promise { }); } -export async function waitForHealthy(port: number) { +export async function waitForIndexedBlock( + port: number, + networkName: string, + blockNumber: number, +) { return new Promise((resolve, reject) => { const timeout = setTimeout(() => { clearInterval(interval); reject(new Error("Timed out while waiting for app to become healthy.")); }, 5_000); const interval = setInterval(async () => { - const response = await fetch(`http://localhost:${port}/health`); + const response = await fetch(`http://localhost:${port}/status`); if (response.status === 200) { - clearTimeout(timeout); - clearInterval(interval); - resolve(undefined); + const status = (await response.json()) as Status; + const statusBlockNumber = status[networkName]?.block?.number; + if ( + statusBlockNumber !== undefined && + statusBlockNumber >= blockNumber + ) { + clearTimeout(timeout); + clearInterval(interval); + resolve(undefined); + } } }, 20); }); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index bada6d461..db854b7ff 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -172,6 +172,11 @@ export async function run({ worker: async (event: RealtimeEvent) => { switch (event.type) { case "newEvents": { + // Note: statusBlocks should be assigned before any other + // asynchronous statements in order to prevent race conditions and + // ensure its correctness. + const statusBlocks = syncService.getStatusBlocks(); + for await (const rawEvents of syncStore.getEvents({ sources, fromCheckpoint: event.fromCheckpoint, @@ -185,7 +190,6 @@ export async function run({ // set status to most recently processed realtime block or end block // for each chain. - const statusBlocks = syncService.getStatusBlocks(); for (const network of networks) { if (statusBlocks[network.name] !== undefined) { status[network.name]!.block = statusBlocks[network.name]!; From 4fdf813912144cad8c1190a3b1865206e130e84c Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 16:24:51 -0400 Subject: [PATCH 083/122] fix: tests --- packages/core/src/_test/utils.ts | 2 +- packages/core/src/bin/utils/run.ts | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/core/src/_test/utils.ts b/packages/core/src/_test/utils.ts index f9fbb3cba..3d5b2a053 100644 --- a/packages/core/src/_test/utils.ts +++ b/packages/core/src/_test/utils.ts @@ -691,7 +691,7 @@ export async function waitForIndexedBlock( return new Promise((resolve, reject) => { const timeout = setTimeout(() => { clearInterval(interval); - reject(new Error("Timed out while waiting for app to become healthy.")); + reject(new Error("Timed out while waiting for the indexed block.")); }, 5_000); const interval = setInterval(async () => { const response = await fetch(`http://localhost:${port}/status`); diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index db854b7ff..220d65523 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -187,18 +187,18 @@ export async function run({ event.toCheckpoint, ); if (result.status === "error") onReloadableError(result.error); + } - // set status to most recently processed realtime block or end block - // for each chain. - for (const network of networks) { - if (statusBlocks[network.name] !== undefined) { - status[network.name]!.block = statusBlocks[network.name]!; - } + // set status to most recently processed realtime block or end block + // for each chain. + for (const network of networks) { + if (statusBlocks[network.name] !== undefined) { + status[network.name]!.block = statusBlocks[network.name]!; } - - await metadataStore.setStatus(status); } + await metadataStore.setStatus(status); + break; } case "reorg": From 5932a69a51f5f86fd48985f19b5067d4958c72fc Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 16:48:32 -0400 Subject: [PATCH 084/122] fix: e2e tests --- packages/core/src/_test/e2e/factory/factory.test.ts | 4 ++-- packages/core/src/sync-realtime/service.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/src/_test/e2e/factory/factory.test.ts b/packages/core/src/_test/e2e/factory/factory.test.ts index 3ac29b09b..2a866c56a 100644 --- a/packages/core/src/_test/e2e/factory/factory.test.ts +++ b/packages/core/src/_test/e2e/factory/factory.test.ts @@ -43,7 +43,7 @@ test("factory", async (context) => { }, }); - await waitForIndexedBlock(port, "mainnet", 4); + await waitForIndexedBlock(port, "mainnet", 5); let response = await postGraphql( port, @@ -74,7 +74,7 @@ test("factory", async (context) => { await simulatePairSwap(context.factory.pair); - await waitForIndexedBlock(port, "mainnet", 5); + await waitForIndexedBlock(port, "mainnet", 6); response = await postGraphql( port, diff --git a/packages/core/src/sync-realtime/service.ts b/packages/core/src/sync-realtime/service.ts index cc312d5d5..93ed36cff 100644 --- a/packages/core/src/sync-realtime/service.ts +++ b/packages/core/src/sync-realtime/service.ts @@ -492,6 +492,8 @@ export const handleBlock = async ( }); } + service.localChain.push(syncBlockToLightBlock(newHeadBlock)); + service.onEvent({ type: "checkpoint", chainId: service.network.chainId, @@ -503,8 +505,6 @@ export const handleBlock = async ( } satisfies Checkpoint, }); - service.localChain.push(syncBlockToLightBlock(newHeadBlock)); - service.common.metrics.ponder_realtime_latest_block_number.set( { network: service.network.name }, newHeadBlockNumber, From 9c73c2460ff9a18cabf9a257c0a8345ecdebc33a Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 17:07:11 -0400 Subject: [PATCH 085/122] fix: race condition w/ status --- packages/core/src/_test/e2e/erc20/erc20.test.ts | 3 +++ packages/core/src/bin/utils/run.ts | 2 +- packages/core/src/sync/service.ts | 9 +++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/core/src/_test/e2e/erc20/erc20.test.ts b/packages/core/src/_test/e2e/erc20/erc20.test.ts index 62c244e11..bc7a26a8c 100644 --- a/packages/core/src/_test/e2e/erc20/erc20.test.ts +++ b/packages/core/src/_test/e2e/erc20/erc20.test.ts @@ -14,6 +14,7 @@ import { import { serve } from "@/bin/commands/serve.js"; import { start } from "@/bin/commands/start.js"; import { range } from "@/utils/range.js"; +import { wait } from "@/utils/wait.js"; import { rimrafSync } from "rimraf"; import { zeroAddress } from "viem"; import { beforeEach, describe, expect, test } from "vitest"; @@ -53,6 +54,8 @@ test("erc20", async (context) => { await waitForIndexedBlock(port, "mainnet", 8); + await wait(1000); + const response = await postGraphql( port, ` diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 220d65523..f1fb889f1 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -175,7 +175,7 @@ export async function run({ // Note: statusBlocks should be assigned before any other // asynchronous statements in order to prevent race conditions and // ensure its correctness. - const statusBlocks = syncService.getStatusBlocks(); + const statusBlocks = syncService.getStatusBlocks(event.toCheckpoint); for await (const rawEvents of syncStore.getEvents({ sources, diff --git a/packages/core/src/sync/service.ts b/packages/core/src/sync/service.ts index f20eafc12..40db7ebc4 100644 --- a/packages/core/src/sync/service.ts +++ b/packages/core/src/sync/service.ts @@ -567,7 +567,10 @@ export const kill = async (syncService: Service) => { }; /** Return the number and timestamp of the most recently processed blocks. */ -export const getStatusBlocks = (syncService: Service) => { +export const getStatusBlocks = ( + syncService: Service, + realtimeCheckpoint?: Checkpoint, +) => { const status: { [networkName: string]: { number: number; timestamp: number } | undefined; } = {}; @@ -581,7 +584,9 @@ export const getStatusBlocks = (syncService: Service) => { } else { const mostRecentBlock = networkService.realtime.realtimeSync.getMostRecentBlock( - syncService.checkpoint, + realtimeCheckpoint === undefined + ? syncService.checkpoint + : checkpointMin(syncService.checkpoint, realtimeCheckpoint), ); if (mostRecentBlock === undefined) { From 8dbb489fe270faf360e9a74eb8c133a8ce1c8d4d Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 17:42:51 -0400 Subject: [PATCH 086/122] delete module cache for hot reloads --- examples/reference-erc20/src/api/index.ts | 1 - packages/core/src/build/service.ts | 4 ++++ packages/core/src/drizzle/runtime.ts | 9 +++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/reference-erc20/src/api/index.ts b/examples/reference-erc20/src/api/index.ts index e4c35118c..017cbdc6a 100644 --- a/examples/reference-erc20/src/api/index.ts +++ b/examples/reference-erc20/src/api/index.ts @@ -6,7 +6,6 @@ ponder.use("/graphql", graphql()).get("/big", async (c) => { const { Account } = c.tables; const account = await c.db - // ^? .select({ balance: Account.balance }) .from(Account) .orderBy(desc(Account.balance)) diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 8d0cc6739..11651fe81 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -283,6 +283,8 @@ export const start = async ( } if (hasSrcUpdate) { + buildService.viteNodeRunner.moduleCache.deleteByModuleId("@/generated"); + const result = await executeIndexingFunctions(buildService); if (result.status === "error") { onBuild({ status: "error", error: result.error }); @@ -371,6 +373,8 @@ export const startServer = async ( .join(", ")}`, }); + buildService.viteNodeRunner.moduleCache.deleteByModuleId("@/generated"); + const result = await executeServer(buildService); if (result.status === "error") { onBuild({ status: "error", error: result.error }); diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index b70290381..f8276dd17 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -138,11 +138,12 @@ export const convertSchemaToDrizzle = ( tableName, drizzleColumns as PostgresTable, ); + } else { + drizzleTables[tableName] = pgSchema(dbNamespace).table( + tableName, + drizzleColumns as PostgresTable, + ); } - drizzleTables[tableName] = pgSchema(dbNamespace).table( - tableName, - drizzleColumns as PostgresTable, - ); } else { drizzleTables[tableName] = sqliteTable( tableName, From 10374033b3f1bf73a2e016b3bdc5152138262d88 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 18:08:24 -0400 Subject: [PATCH 087/122] fix: dev server reloading --- packages/core/src/bin/commands/dev.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/core/src/bin/commands/dev.ts b/packages/core/src/bin/commands/dev.ts index e910ed673..f75e1b0dd 100644 --- a/packages/core/src/bin/commands/dev.ts +++ b/packages/core/src/bin/commands/dev.ts @@ -1,7 +1,7 @@ import { existsSync } from "node:fs"; import path from "node:path"; import { type BuildResult, createBuildService } from "@/build/index.js"; -import type { Build, BuildResultServer } from "@/build/service.js"; +import type { Build, BuildResultServer, BuildServer } from "@/build/service.js"; import { createLogger } from "@/common/logger.js"; import { MetricsService } from "@/common/metrics.js"; import { buildOptions } from "@/common/options.js"; @@ -68,6 +68,7 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); let cachedBuild: Build; + let cachedBuildServer: BuildServer | undefined; const buildQueue = createQueue({ initialStart: true, @@ -97,6 +98,12 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { buildQueue.add({ type: "indexing", status: "error", error }); }, }); + + cleanupReloadableServer = await runServer({ + common, + build: cachedBuild, + buildServer: cachedBuildServer, + }); } else { // This handles build failures and indexing errors on hot reload. uiService.setReloadableError(); @@ -106,6 +113,8 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { await cleanupReloadableServer(); if (result.status === "success") { + cachedBuildServer = result.build; + cleanupReloadableServer = await runServer({ common, build: cachedBuild, @@ -148,6 +157,9 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { properties: { cli_command: "dev", ...buildPayload(initialResult.build) }, }); + cachedBuild = initialResult.build; + cachedBuildServer = initialResultServer.build; + buildQueue .add({ type: "indexing", ...initialResult }) .then(() => buildQueue.add({ type: "server", ...initialResultServer })); From 321728e7e3a4f2fbc198ca0794e2dc60bcdf3b13 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 18:26:42 -0400 Subject: [PATCH 088/122] chore: cleanup e2e test --- packages/core/src/_test/e2e/erc20/erc20.test.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/core/src/_test/e2e/erc20/erc20.test.ts b/packages/core/src/_test/e2e/erc20/erc20.test.ts index bc7a26a8c..62c244e11 100644 --- a/packages/core/src/_test/e2e/erc20/erc20.test.ts +++ b/packages/core/src/_test/e2e/erc20/erc20.test.ts @@ -14,7 +14,6 @@ import { import { serve } from "@/bin/commands/serve.js"; import { start } from "@/bin/commands/start.js"; import { range } from "@/utils/range.js"; -import { wait } from "@/utils/wait.js"; import { rimrafSync } from "rimraf"; import { zeroAddress } from "viem"; import { beforeEach, describe, expect, test } from "vitest"; @@ -54,8 +53,6 @@ test("erc20", async (context) => { await waitForIndexedBlock(port, "mainnet", 8); - await wait(1000); - const response = await postGraphql( port, ` From b188f6f55f146c295b4134c20efaf26bc64e2df0 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 18:35:52 -0400 Subject: [PATCH 089/122] fix metadata for graphql and create graphql endpoint when no custom endpoint exists --- packages/core/src/graphql/index.ts | 3 ++- packages/core/src/server/service.ts | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/core/src/graphql/index.ts b/packages/core/src/graphql/index.ts index d496fbef0..f9040f99f 100644 --- a/packages/core/src/graphql/index.ts +++ b/packages/core/src/graphql/index.ts @@ -26,6 +26,7 @@ export const graphql = ( ) => createMiddleware(async (c) => { const readonlyStore = c.get("readonlyStore"); + const metadataStore = c.get("metadataStore"); const schema = c.get("schema"); const graphqlSchema = buildGraphQLSchema(schema); @@ -37,7 +38,7 @@ export const graphql = ( schema: graphqlSchema, context: () => { const getLoader = buildLoaderCache({ store: readonlyStore }); - return { readonlyStore, getLoader }; + return { readonlyStore, metadataStore, getLoader }; }, graphqlEndpoint: c.req.path, maskedErrors: process.env.NODE_ENV === "production", diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 82acd3245..6b20f1307 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -2,6 +2,7 @@ import http from "node:http"; import type { Common } from "@/common/common.js"; import type { DatabaseService } from "@/database/service.js"; import { convertSchemaToDrizzle, createDrizzleDb } from "@/drizzle/runtime.js"; +import { graphql } from "@/graphql/index.js"; import { type PonderRoutes, applyHonoRoutes } from "@/hono/index.js"; import { getMetadataStore } from "@/indexing-store/metadata.js"; import { getReadonlyStore } from "@/indexing-store/readonly.js"; @@ -131,6 +132,7 @@ export async function createServer({ // context required for graphql middleware const contextMiddleware = createMiddleware(async (c, next) => { c.set("readonlyStore", readonlyStore); + c.set("metadataStore", metadataStore); c.set("schema", schema); await next(); }); @@ -168,6 +170,10 @@ export async function createServer({ onError(error, c, common), ), ); + } else { + // apply graphql middleware if no custom api exists + hono.use("/graphql", graphql()); + hono.use("/", graphql()); } // Create nodejs server From 40e8a9b892e101e133c15ba247c559c874723b98 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Fri, 12 Jul 2024 18:52:08 -0400 Subject: [PATCH 090/122] rename metadata table --- .../src/database/postgres/service.test.ts | 34 +++++++++---------- .../core/src/database/postgres/service.ts | 6 ++-- .../core/src/database/sqlite/service.test.ts | 14 ++++---- packages/core/src/database/sqlite/service.ts | 4 +-- packages/core/src/indexing-store/metadata.ts | 8 ++--- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/packages/core/src/database/postgres/service.test.ts b/packages/core/src/database/postgres/service.test.ts index ab282a5f7..8e8f8a5f2 100644 --- a/packages/core/src/database/postgres/service.test.ts +++ b/packages/core/src/database/postgres/service.test.ts @@ -79,7 +79,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { ]); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", ]); @@ -114,7 +114,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", ]); @@ -129,7 +129,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Dog", "Apple", ]); @@ -161,14 +161,14 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", ]); expect(await getViewNames(databaseTwo.db, "publish")).toStrictEqual([ "Pet", "Person", - "_ponder_metadata", + "_ponder_meta", ]); await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); @@ -182,14 +182,14 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Dog", "Apple", ]); expect(await getViewNames(databaseTwo.db, "publish")).toStrictEqual([ "Dog", "Apple", - "_ponder_metadata", + "_ponder_meta", ]); await databaseTwo.kill(); @@ -222,7 +222,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { ); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", "not_a_ponder_table", @@ -232,7 +232,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "not_a_ponder_table", "AnotherTable", "Dog", @@ -571,7 +571,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public2", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Dog", "Apple", ]); @@ -616,7 +616,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { hash(["public2", "abc", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Dog", "Apple", ]); @@ -637,7 +637,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", ]); @@ -647,7 +647,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getViewNames(database.db, "publish")).toStrictEqual([ "Pet", "Person", - "_ponder_metadata", + "_ponder_meta", ]); await database.kill(); @@ -665,7 +665,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", ]); @@ -680,7 +680,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getTableNames(database.db, "publish")).toStrictEqual(["Pet"]); expect(await getViewNames(database.db, "publish")).toStrictEqual([ "Person", - "_ponder_metadata", + "_ponder_meta", ]); await database.kill(); @@ -698,7 +698,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { await database.setup({ schema, buildId: "abc" }); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", ]); @@ -718,7 +718,7 @@ describe.skipIf(shouldSkip)("postgres database", () => { expect(await getViewNames(database.db, "nice_looks-great")).toStrictEqual([ "Pet", "Person", - "_ponder_metadata", + "_ponder_meta", ]); await database.kill(); diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index ad3721bf8..d00844bf5 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -265,7 +265,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { // Create ponder_metadata table if it doesn't exist await tx.schema .withSchema(this.userNamespace) - .createTable("_ponder_metadata") + .createTable("_ponder_meta") .addColumn("key", "text", (col) => col.primaryKey()) .addColumn("value", "jsonb") .ifNotExists() @@ -275,7 +275,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { await tx .withSchema(this.userNamespace) // @ts-expect-error Kysely doesn't have types for user schema - .insertInto("_ponder_metadata") + .insertInto("_ponder_meta") // @ts-expect-error Kysely doesn't have types for user schema .values({ key: "status", value: null }) // @ts-expect-error Kysely doesn't have types for user schema @@ -589,7 +589,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { await tx.schema.createSchema(publishSchema).ifNotExists().execute(); for (const tableName of Object.keys(getTables(this.schema)).concat( - "_ponder_metadata", + "_ponder_meta", )) { // Check if there is an existing relation with the name we're about to publish. const result = await tx.executeQuery<{ diff --git a/packages/core/src/database/sqlite/service.test.ts b/packages/core/src/database/sqlite/service.test.ts index c8b686732..175a8afb8 100644 --- a/packages/core/src/database/sqlite/service.test.ts +++ b/packages/core/src/database/sqlite/service.test.ts @@ -78,7 +78,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { ]); expect(await getTableNames(database.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", ]); @@ -111,7 +111,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public", "abc", "Person"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", ]); @@ -126,7 +126,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Dog", "Apple", ]); @@ -159,7 +159,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { ); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Pet", "Person", "not_a_ponder_table", @@ -169,7 +169,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { await databaseTwo.setup({ schema: schemaTwo, buildId: "def" }); expect(await getTableNames(databaseTwo.db, "public")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "not_a_ponder_table", "AnotherTable", "Dog", @@ -494,7 +494,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public2", "def", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Dog", "Apple", ]); @@ -538,7 +538,7 @@ describe.skipIf(shouldSkip)("sqlite database", () => { hash(["public2", "abc", "Apple"]), ]); expect(await getTableNames(databaseTwo.db, "public2")).toStrictEqual([ - "_ponder_metadata", + "_ponder_meta", "Dog", "Apple", ]); diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index e7c036693..02d0f178f 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -236,7 +236,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { // Create ponder_metadata table if it doesn't exist await tx.schema .withSchema(this.userNamespace) - .createTable("_ponder_metadata") + .createTable("_ponder_meta") .addColumn("key", "text", (col) => col.primaryKey()) .addColumn("value", "jsonb") .ifNotExists() @@ -246,7 +246,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { await tx .withSchema(this.userNamespace) // @ts-expect-error Kysely doesn't have types for user schema - .insertInto("_ponder_metadata") + .insertInto("_ponder_meta") // @ts-expect-error Kysely doesn't have types for user schema .values({ key: "status", value: null }) // @ts-expect-error Kysely doesn't have types for user schema diff --git a/packages/core/src/indexing-store/metadata.ts b/packages/core/src/indexing-store/metadata.ts index c59701b2d..d5ec764bd 100644 --- a/packages/core/src/indexing-store/metadata.ts +++ b/packages/core/src/indexing-store/metadata.ts @@ -12,10 +12,10 @@ export const getMetadataStore = ({ db: HeadlessKysely; }): MetadataStore => ({ getStatus: async () => { - return db.wrap({ method: "_ponder_metadata.getLatest()" }, async () => { + return db.wrap({ method: "_ponder_meta.getLatest()" }, async () => { const metadata = await db .withSchema(namespaceInfo.userNamespace) - .selectFrom("_ponder_metadata") + .selectFrom("_ponder_meta") .select("value") .where("key", "=", "status") .executeTakeFirst(); @@ -28,10 +28,10 @@ export const getMetadataStore = ({ }); }, setStatus: (status: Status) => { - return db.wrap({ method: "_ponder_metadata.setLatest()" }, async () => { + return db.wrap({ method: "_ponder_meta.setLatest()" }, async () => { await db .withSchema(namespaceInfo.userNamespace) - .insertInto("_ponder_metadata") + .insertInto("_ponder_meta") .values({ key: "status", value: encoding === "sqlite" ? JSON.stringify(status) : status, From b9d8955dd89aee5809297350ca4b35ae25ba0b92 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 11:50:58 -0400 Subject: [PATCH 091/122] fix e2e tests --- packages/core/src/_test/utils.ts | 6 ++++-- packages/core/src/indexing-store/metadata.ts | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/core/src/_test/utils.ts b/packages/core/src/_test/utils.ts index cc54fca76..5cb9edb50 100644 --- a/packages/core/src/_test/utils.ts +++ b/packages/core/src/_test/utils.ts @@ -696,8 +696,10 @@ export async function waitForIndexedBlock( const interval = setInterval(async () => { const response = await fetch(`http://localhost:${port}/_ponder/status`); if (response.status === 200) { - const status = (await response.json()) as Status; - const statusBlockNumber = status[networkName]?.block?.number; + const status = (await response.json()) as Status | null; + const statusBlockNumber = status + ? status[networkName]?.block?.number + : undefined; if ( statusBlockNumber !== undefined && statusBlockNumber >= blockNumber diff --git a/packages/core/src/indexing-store/metadata.ts b/packages/core/src/indexing-store/metadata.ts index c3c00e582..cff041587 100644 --- a/packages/core/src/indexing-store/metadata.ts +++ b/packages/core/src/indexing-store/metadata.ts @@ -12,7 +12,7 @@ export const getMetadataStore = ({ db: HeadlessKysely; }): MetadataStore => ({ getStatus: async () => { - return db.wrap({ method: "_ponder_meta.getLatest()" }, async () => { + return db.wrap({ method: "_ponder_meta.getStatus()" }, async () => { const metadata = await db .withSchema(namespaceInfo.userNamespace) .selectFrom("_ponder_meta") @@ -28,7 +28,7 @@ export const getMetadataStore = ({ }); }, setStatus: (status: Status) => { - return db.wrap({ method: "_ponder_meta.setLatest()" }, async () => { + return db.wrap({ method: "_ponder_meta.setStatus()" }, async () => { await db .withSchema(namespaceInfo.userNamespace) .insertInto("_ponder_meta") From 9181ad180479fdad0f990a30b96e936065ba61f8 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 11:55:45 -0400 Subject: [PATCH 092/122] fix server tests --- packages/core/src/server/service.test.ts | 32 ++++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/core/src/server/service.test.ts b/packages/core/src/server/service.test.ts index 3f9cb1390..cfd756627 100644 --- a/packages/core/src/server/service.test.ts +++ b/packages/core/src/server/service.test.ts @@ -12,7 +12,8 @@ beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); test("port", async (context) => { - const { database, namespaceInfo } = await setupDatabaseServices(context); + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); const server1 = await createServer({ common: context.common, @@ -32,10 +33,12 @@ test("port", async (context) => { await server1.kill(); await server2.kill(); + await cleanup(); }); test("not healthy", async (context) => { - const { database, namespaceInfo } = await setupDatabaseServices(context); + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); const server = await createServer({ common: context.common, @@ -49,10 +52,12 @@ test("not healthy", async (context) => { expect(response.status).toBe(503); await server.kill(); + await cleanup(); }); test("healthy", async (context) => { - const { database, namespaceInfo } = await setupDatabaseServices(context); + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); const server = await createServer({ common: context.common, @@ -72,10 +77,12 @@ test("healthy", async (context) => { expect(response.status).toBe(200); await server.kill(); + await cleanup(); }); test("healthy PUT", async (context) => { - const { database, namespaceInfo } = await setupDatabaseServices(context); + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); const server = await createServer({ common: { @@ -94,10 +101,12 @@ test("healthy PUT", async (context) => { expect(response.status).toBe(404); await server.kill(); + await cleanup(); }); test("metrics", async (context) => { - const { database, namespaceInfo } = await setupDatabaseServices(context); + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); const server = await createServer({ common: context.common, @@ -111,10 +120,12 @@ test("metrics", async (context) => { expect(response.status).toBe(200); await server.kill(); + await cleanup(); }); test("metrics error", async (context) => { - const { database, namespaceInfo } = await setupDatabaseServices(context); + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); const server = await createServer({ common: context.common, @@ -131,10 +142,12 @@ test("metrics error", async (context) => { expect(response.status).toBe(500); await server.kill(); + await cleanup(); }); test("metrics PUT", async (context) => { - const { database, namespaceInfo } = await setupDatabaseServices(context); + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); const server = await createServer({ common: context.common, @@ -150,10 +163,12 @@ test("metrics PUT", async (context) => { expect(response.status).toBe(404); await server.kill(); + await cleanup(); }); test("missing route", async (context) => { - const { database, namespaceInfo } = await setupDatabaseServices(context); + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); const server = await createServer({ common: context.common, @@ -167,6 +182,7 @@ test("missing route", async (context) => { expect(response.status).toBe(404); await server.kill(); + await cleanup(); }); // Note that this test doesn't work because the `hono.request` method doesn't actually From 81ce418152381e9bd8b688d07f535f54e14e7e7d Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 12:21:37 -0400 Subject: [PATCH 093/122] cleanup Signed-off-by: Kyle Scott --- packages/core/src/bin/utils/runServer.ts | 1 - packages/core/src/graphql/index.ts | 10 ++++++++++ packages/core/src/hono/context.ts | 12 ------------ 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/core/src/bin/utils/runServer.ts b/packages/core/src/bin/utils/runServer.ts index 7cffbdaff..88ae1eca2 100644 --- a/packages/core/src/bin/utils/runServer.ts +++ b/packages/core/src/bin/utils/runServer.ts @@ -49,6 +49,5 @@ export async function runServer({ return async () => { await server.kill(); - await database.kill(); }; } diff --git a/packages/core/src/graphql/index.ts b/packages/core/src/graphql/index.ts index f9040f99f..009246931 100644 --- a/packages/core/src/graphql/index.ts +++ b/packages/core/src/graphql/index.ts @@ -7,6 +7,16 @@ import { createMiddleware } from "hono/factory"; import { buildGraphQLSchema } from "./buildGraphqlSchema.js"; import { buildLoaderCache } from "./buildLoaderCache.js"; +/** + * Middleware for GraphQL with an interactive web view. + * + * - Docs: [TODO(kyle)] + * + * @example + * import { ponder } from "@/generated"; + * ponder.use("/graphql", graphql()); + * + */ export const graphql = ( { maxOperationTokens = 1000, diff --git a/packages/core/src/hono/context.ts b/packages/core/src/hono/context.ts index 818280965..65901ea5a 100644 --- a/packages/core/src/hono/context.ts +++ b/packages/core/src/hono/context.ts @@ -8,13 +8,7 @@ export type Context< path extends string = string, input extends Input = {}, > = { - /** - * ... - */ db: DrizzleDb; - /** - * - */ tables: { [tableName in ExtractTableNames]: ConvertToDrizzleTable< tableName, @@ -66,13 +60,7 @@ export type MiddlewareContext< path extends string = string, input extends Input = {}, > = HonoContext & { - /** - * ... - */ db: DrizzleDb; - /** - * - */ tables: { [tableName in ExtractTableNames]: ConvertToDrizzleTable< tableName, From 377af20a3a10278168f92f35dc7440f00482118e Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 15:17:53 -0400 Subject: [PATCH 094/122] fix: hot reloading --- packages/core/src/bin/commands/dev.ts | 7 ++- packages/core/src/build/service.ts | 70 ++++++++++++++++----------- packages/core/src/server/service.ts | 22 ++++----- 3 files changed, 54 insertions(+), 45 deletions(-) diff --git a/packages/core/src/bin/commands/dev.ts b/packages/core/src/bin/commands/dev.ts index f75e1b0dd..4a2bf5e6b 100644 --- a/packages/core/src/bin/commands/dev.ts +++ b/packages/core/src/bin/commands/dev.ts @@ -80,6 +80,7 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { ) => { if (result.type === "indexing") { await cleanupReloadable(); + await cleanupReloadableServer(); if (result.status === "success") { uiService.reset(); @@ -123,7 +124,7 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { } else { // This handles build failures and indexing errors on hot reload. uiService.setReloadableError(); - cleanupReloadable = () => Promise.resolve(); + cleanupReloadableServer = () => Promise.resolve(); } } }, @@ -160,9 +161,7 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { cachedBuild = initialResult.build; cachedBuildServer = initialResultServer.build; - buildQueue - .add({ type: "indexing", ...initialResult }) - .then(() => buildQueue.add({ type: "server", ...initialResultServer })); + buildQueue.add({ type: "indexing", ...initialResult }); return async () => { buildQueue.pause(); diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 11651fe81..af1d193ec 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -16,7 +16,6 @@ import { type ViteDevServer, createServer } from "vite"; import { ViteNodeRunner } from "vite-node/client"; import { ViteNodeServer } from "vite-node/server"; import { installSourcemapsSupport } from "vite-node/source-map"; -import { normalizeModuleId, toFilePath } from "vite-node/utils"; import viteTsconfigPathsPlugin from "vite-tsconfig-paths"; import { type IndexingFunctions, @@ -34,6 +33,8 @@ export type Service = { common: Common; srcRegex: RegExp; serverRegex: RegExp; + srcPattern: string; + serverPattern: string; // vite viteDevServer: ViteDevServer; @@ -99,6 +100,15 @@ export const create = async ({ .replace(escapeRegex, "\\$&"); const serverRegex = new RegExp(`^${escapedServerDir}/.*\\.(ts|js)$`); + // TODO(kyle) ignore server files + const srcPattern = path + .join(common.options.srcDir, "**/*.{js,mjs,ts,mts}") + .replace(/\\/g, "/"); + + const serverPattern = path + .join(common.options.serverDir, "**/*.{js,mjs,ts,mts}") + .replace(/\\/g, "/"); + const viteLogger = { warnedMessages: new Set(), loggedErrors: new WeakSet(), @@ -151,6 +161,8 @@ export const create = async ({ common, srcRegex, serverRegex, + srcPattern, + serverPattern, viteDevServer, viteNodeServer, viteNodeRunner, @@ -218,19 +230,19 @@ export const start = async ( return isInIgnoredDir || isIgnoredFile; }; - const onFileChange = async (_file: string) => { - if (isFileIgnored(_file)) return; + const onFileChange = async (file: string) => { + if (isFileIgnored(file)) return; - // Note that `toFilePath` always returns a POSIX path, even if you pass a Windows path. - const file = toFilePath( - normalizeModuleId(_file), - common.options.rootDir, - ).path; + const files = glob.sync(buildService.srcPattern, { + ignore: buildService.serverPattern, + }); + + if (files.includes(file) === false) return; // Invalidate all modules that depend on the updated files. // Note that `invalidateDepTree` accepts and returns POSIX paths, even on Windows. const invalidated = [ - ...buildService.viteNodeRunner.moduleCache.invalidateDepTree([file]), + ...buildService.viteNodeRunner.moduleCache.invalidateDepTree(files), ]; // If no files were invalidated, no need to reload. @@ -340,19 +352,17 @@ export const startServer = async ( return isInIgnoredDir || isIgnoredFile; }; - const onFileChange = async (_file: string) => { - if (isFileIgnored(_file)) return; + const onFileChange = async (file: string) => { + if (isFileIgnored(file)) return; - // Note that `toFilePath` always returns a POSIX path, even if you pass a Windows path. - const file = toFilePath( - normalizeModuleId(_file), - common.options.rootDir, - ).path; + const files = glob.sync(buildService.serverPattern); - // Invalidate all modules that depend on the updated files. + if (files.includes(file) === false) return; + + // Invalidate all server modules. // Note that `invalidateDepTree` accepts and returns POSIX paths, even on Windows. const invalidated = [ - ...buildService.viteNodeRunner.moduleCache.invalidateDepTree([file]), + ...buildService.viteNodeRunner.moduleCache.invalidateDepTree(files), ]; // If no files were invalidated, no need to reload. @@ -376,10 +386,19 @@ export const startServer = async ( buildService.viteNodeRunner.moduleCache.deleteByModuleId("@/generated"); const result = await executeServer(buildService); + if (result.status === "error") { onBuild({ status: "error", error: result.error }); return; } + + onBuild({ + status: result.status, + build: + result.app && result.routes + ? { app: result.app, routes: result.routes } + : undefined, + }); }; buildService.viteDevServer.watcher.on("change", onFileChange); @@ -467,12 +486,9 @@ const executeIndexingFunctions = async ( } | { status: "error"; error: Error } > => { - // TODO(kyle) ignore server file - const pattern = path - .join(buildService.common.options.srcDir, "**/*.{js,mjs,ts,mts}") - .replace(/\\/g, "/"); - const files = glob.sync(pattern); - + const files = glob.sync(buildService.srcPattern, { + ignore: buildService.serverPattern, + }); const executeResults = await Promise.all( files.map(async (file) => ({ ...(await executeFile(buildService, { file })), @@ -537,11 +553,7 @@ const executeServer = async ( return { status: "success" }; } - const pattern = path - .join(buildService.common.options.serverDir, "**/*.{js,mjs,ts,mts}") - .replace(/\\/g, "/"); - const files = glob.sync(pattern); - + const files = glob.sync(buildService.serverPattern); const executeResults = await Promise.all( files.map(async (file) => ({ ...(await executeFile(buildService, { file })), diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 6b20f1307..8e3b61a93 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -153,23 +153,21 @@ export async function createServer({ } } - common.logger.debug({ - service: "server", - msg: `Detected a custom server with routes: [${userApp.routes - .map((r) => r.path) - .join(", ")}]`, - }); - const db = createDrizzleDb(database); const tables = convertSchemaToDrizzle(schema, database, dbNamespace); // apply user routes to hono instance, registering a custom error handler - hono.route( - "/", - applyHonoRoutes(userApp, userRoutes, { db, tables }).onError((error, c) => - onError(error, c, common), - ), + applyHonoRoutes(hono, userRoutes, { db, tables }).onError((error, c) => + onError(error, c, common), ); + + common.logger.debug({ + service: "server", + msg: `Detected a custom server with routes: [${userRoutes + .map(({ pathOrHandlers: [maybePathOrHandler] }) => maybePathOrHandler) + .filter((maybePathOrHandler) => typeof maybePathOrHandler === "string") + .join(", ")}]`, + }); } else { // apply graphql middleware if no custom api exists hono.use("/graphql", graphql()); From ce0077f2fefc8671e22a707d82c6361cf94e3715 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 15:34:40 -0400 Subject: [PATCH 095/122] add more drizzle tests Signed-off-by: Kyle Scott --- packages/core/src/drizzle/table.test-d.ts | 45 +++++++++++++++++++++++ packages/core/src/drizzle/table.ts | 9 ++++- packages/core/src/schema/common.ts | 1 + 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/packages/core/src/drizzle/table.test-d.ts b/packages/core/src/drizzle/table.test-d.ts index 7e1f6aa0c..5298dc9d6 100644 --- a/packages/core/src/drizzle/table.test-d.ts +++ b/packages/core/src/drizzle/table.test-d.ts @@ -45,6 +45,51 @@ test("select optional column", async () => { expectTypeOf<{ id: string; name: number | null }[]>(result); }); +test("select enum", async () => { + const schema = createSchema((p) => ({ + e: p.createEnum(["yes", "no"]), + table: p.createTable({ + id: p.string(), + name: p.int().optional(), + e: p.enum("e"), + }), + })); + + const table = {} as ConvertToDrizzleTable< + "table", + (typeof schema)["table"]["table"], + typeof schema + >; + + const result = await ({} as DrizzleDb).select().from(table); + // ^? + + expectTypeOf<{ id: string; name: number | null; e: "yes" | "no" }[]>(result); +}); + +test("select json", async () => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.string(), + name: p.int().optional(), + json: p.json<{ a: number; b: string }>(), + }), + })); + + const table = {} as ConvertToDrizzleTable< + "table", + (typeof schema)["table"]["table"], + typeof schema + >; + + const result = await ({} as DrizzleDb).select().from(table); + // ^? + + expectTypeOf< + { id: string; name: number | null; json: { a: number; b: string } }[] + >(result); +}); + test("select join", async () => { const schema = createSchema((p) => ({ account: p.createTable({ diff --git a/packages/core/src/drizzle/table.ts b/packages/core/src/drizzle/table.ts index 2fc307f09..4ee46d48d 100644 --- a/packages/core/src/drizzle/table.ts +++ b/packages/core/src/drizzle/table.ts @@ -1,5 +1,7 @@ import type { + EnumColumn, ExtractNonVirtualColumnNames, + JSONColumn, Schema as PonderSchema, Table as PonderTable, ReferenceColumn, @@ -32,7 +34,12 @@ export type ConvertToDrizzleTable< driverParam: unknown; enumValues: undefined; notNull: (table[columnName] & - (ScalarColumn | ReferenceColumn))[" optional"] extends true + ( + | ScalarColumn + | ReferenceColumn + | EnumColumn + | JSONColumn + ))[" optional"] extends true ? false : true; primaryKey: columnName extends "id" ? true : false; diff --git a/packages/core/src/schema/common.ts b/packages/core/src/schema/common.ts index 17422773a..f749720ef 100644 --- a/packages/core/src/schema/common.ts +++ b/packages/core/src/schema/common.ts @@ -189,6 +189,7 @@ export type ExtractNonVirtualColumnNames< | ReferenceColumn | ScalarColumn | EnumColumn + | JSONColumn ? columnNames : never : never; From 4a2681e6fb834d26e8d6167f5445b4ffc054ca12 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 15:39:07 -0400 Subject: [PATCH 096/122] jsdoc --- packages/core/src/drizzle/db.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/core/src/drizzle/db.ts b/packages/core/src/drizzle/db.ts index 9f6f108db..1c5013652 100644 --- a/packages/core/src/drizzle/db.ts +++ b/packages/core/src/drizzle/db.ts @@ -9,6 +9,20 @@ export type DrizzleDb = { select( fields?: SelectedFields, ): SelectBuilder | undefined, "async", void>; + /** + * Execute raw, readonly sql queries. + * + * @example + * import { ponder } from "@/generated"; + * import { sql } from "@ponder/core"; + * + * ponder.get("/", async (c) => { + * const result = await c.db.execute(sql`SELECT * from "Accounts" `); + * return c.json(result); + * }); + * + * @see https://orm.drizzle.team/docs/sql + */ execute: >( query: SQLWrapper, ) => Promise; From d99266d0e6a33f412ea7c9176aa248b6658fad27 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 16:05:41 -0400 Subject: [PATCH 097/122] cleanup --- packages/core/src/build/service.ts | 3 ++- packages/core/src/drizzle/db.ts | 2 +- packages/core/src/server/error.ts | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index af1d193ec..85d15b52c 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -100,7 +100,6 @@ export const create = async ({ .replace(escapeRegex, "\\$&"); const serverRegex = new RegExp(`^${escapedServerDir}/.*\\.(ts|js)$`); - // TODO(kyle) ignore server files const srcPattern = path .join(common.options.srcDir, "**/*.{js,mjs,ts,mts}") .replace(/\\/g, "/"); @@ -359,6 +358,8 @@ export const startServer = async ( if (files.includes(file) === false) return; + // TODO(kyle) maybe remove some lines? + // Invalidate all server modules. // Note that `invalidateDepTree` accepts and returns POSIX paths, even on Windows. const invalidated = [ diff --git a/packages/core/src/drizzle/db.ts b/packages/core/src/drizzle/db.ts index 1c5013652..d12fe7fd6 100644 --- a/packages/core/src/drizzle/db.ts +++ b/packages/core/src/drizzle/db.ts @@ -17,7 +17,7 @@ export type DrizzleDb = { * import { sql } from "@ponder/core"; * * ponder.get("/", async (c) => { - * const result = await c.db.execute(sql`SELECT * from "Accounts" `); + * const result = await c.db.execute(sql`SELECT * from "Accounts"`); * return c.json(result); * }); * diff --git a/packages/core/src/server/error.ts b/packages/core/src/server/error.ts index 4813f4385..56976ec9c 100644 --- a/packages/core/src/server/error.ts +++ b/packages/core/src/server/error.ts @@ -55,7 +55,6 @@ export const onNotFound = (c: Context) => { }; const tryExtractRequestBody = async (request: HonoRequest) => { - // try { return await request.json(); } catch { From 40b2ba0f32bf2e08b44ec00c266093635123b223 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 16:20:32 -0400 Subject: [PATCH 098/122] cleanup hot reloading --- packages/core/src/bin/commands/dev.ts | 3 ++ packages/core/src/build/service.ts | 41 ++++++++++----------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/packages/core/src/bin/commands/dev.ts b/packages/core/src/bin/commands/dev.ts index 4a2bf5e6b..d97ef10c8 100644 --- a/packages/core/src/bin/commands/dev.ts +++ b/packages/core/src/bin/commands/dev.ts @@ -70,6 +70,9 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { let cachedBuild: Build; let cachedBuildServer: BuildServer | undefined; + // Note: an update to the "indexing" build triggers a reload of the server runtime. + // This is to ensure that updates to either the config or schema are reflected in + // the server. const buildQueue = createQueue({ initialStart: true, concurrency: 1, diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index 85d15b52c..d9f67472b 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -16,6 +16,7 @@ import { type ViteDevServer, createServer } from "vite"; import { ViteNodeRunner } from "vite-node/client"; import { ViteNodeServer } from "vite-node/server"; import { installSourcemapsSupport } from "vite-node/source-map"; +import { normalizeModuleId, toFilePath } from "vite-node/utils"; import viteTsconfigPathsPlugin from "vite-tsconfig-paths"; import { type IndexingFunctions, @@ -229,19 +230,19 @@ export const start = async ( return isInIgnoredDir || isIgnoredFile; }; - const onFileChange = async (file: string) => { - if (isFileIgnored(file)) return; + const onFileChange = async (_file: string) => { + if (isFileIgnored(_file)) return; - const files = glob.sync(buildService.srcPattern, { - ignore: buildService.serverPattern, - }); - - if (files.includes(file) === false) return; + // Note that `toFilePath` always returns a POSIX path, even if you pass a Windows path. + const file = toFilePath( + normalizeModuleId(_file), + common.options.rootDir, + ).path; // Invalidate all modules that depend on the updated files. // Note that `invalidateDepTree` accepts and returns POSIX paths, even on Windows. const invalidated = [ - ...buildService.viteNodeRunner.moduleCache.invalidateDepTree(files), + ...buildService.viteNodeRunner.moduleCache.invalidateDepTree([file]), ]; // If no files were invalidated, no need to reload. @@ -294,6 +295,10 @@ export const start = async ( } if (hasSrcUpdate) { + const files = glob.sync(buildService.srcPattern, { + ignore: buildService.serverPattern, + }); + buildService.viteNodeRunner.moduleCache.invalidateDepTree(files); buildService.viteNodeRunner.moduleCache.deleteByModuleId("@/generated"); const result = await executeIndexingFunctions(buildService); @@ -358,28 +363,12 @@ export const startServer = async ( if (files.includes(file) === false) return; - // TODO(kyle) maybe remove some lines? - // Invalidate all server modules. - // Note that `invalidateDepTree` accepts and returns POSIX paths, even on Windows. - const invalidated = [ - ...buildService.viteNodeRunner.moduleCache.invalidateDepTree(files), - ]; - - // If no files were invalidated, no need to reload. - if (invalidated.length === 0) return; - - const hasServerUpdate = invalidated.some((file) => - buildService.serverRegex.test(file), - ); - - // This branch could trigger if you change a `note.txt` file within `src/`. - // Note: We could probably do a better job filtering out files in `isFileIgnored`. - if (!hasServerUpdate) return; + buildService.viteNodeRunner.moduleCache.invalidateDepTree(files); common.logger.info({ service: "build", - msg: `Hot reload ${invalidated + msg: `Hot reload ${files .map((f) => `'${path.relative(common.options.rootDir, f)}'`) .join(", ")}`, }); From 12fec053d144908b229d6caf94c9d7dec924cfe0 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 17:16:06 -0400 Subject: [PATCH 099/122] add server validation at build step and remove /_ponder from internal api routes --- packages/core/src/build/service.ts | 58 ++++++++++++++++----- packages/core/src/server/service.ts | 81 ++++++++++++----------------- 2 files changed, 78 insertions(+), 61 deletions(-) diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index d9f67472b..f16ff7541 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import type { Common } from "@/common/common.js"; +import { BuildError } from "@/common/errors.js"; import type { Config, OptionsConfig } from "@/config/config.js"; import type { DatabaseConfig } from "@/config/database.js"; import type { Network } from "@/config/networks.js"; @@ -330,12 +331,14 @@ export const startServer = async ( ): Promise => { const { common } = buildService; - const buildResult = await executeServer(buildService); + const serverResult = await executeServer(buildService); - if (buildResult.status === "error") { - return { status: "error", error: buildResult.error }; + if (serverResult.status === "error") { + return { status: "error", error: serverResult.error }; } + const buildResult = validateAndBuildServer(buildService, serverResult); + // If watch is false (`ponder start` or `ponder serve`), // don't register any event handlers on the watcher. if (watch) { @@ -382,22 +385,13 @@ export const startServer = async ( return; } - onBuild({ - status: result.status, - build: - result.app && result.routes - ? { app: result.app, routes: result.routes } - : undefined, - }); + onBuild(validateAndBuildServer(buildService, result)); }; buildService.viteDevServer.watcher.on("change", onFileChange); } - return { - status: "success", - build: buildResult.app ? (buildResult as BuildServer) : undefined, - }; + return buildResult; }; export const kill = async (buildService: Service): Promise => { @@ -649,6 +643,42 @@ const validateAndBuild = async ( }; }; +const validateAndBuildServer = ( + { common }: Pick, + build: Partial, +): BuildResultServer => { + if (!build.app || !build.routes) { + return { status: "success" }; + } + + for (const { + pathOrHandlers: [maybePathOrHandler], + } of build.routes) { + if (typeof maybePathOrHandler === "string") { + if ( + maybePathOrHandler === "/status" || + maybePathOrHandler === "/metrics" || + maybePathOrHandler === "/health" + ) { + const error = new BuildError( + `Validation failed: API route "${maybePathOrHandler}" is reserved for internal use.`, + ); + error.stack = undefined; + common.logger.error({ service: "build", msg: "Failed build", error }); + return { status: "error", error }; + } + } + } + + return { + status: "success", + build: { + app: build.app, + routes: build.routes, + }, + }; +}; + const executeFile = async ( { common, viteNodeRunner }: Service, { file }: { file: string }, diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 8e3b61a93..3df8f1d6f 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -54,45 +54,6 @@ export async function createServer({ db: database.readonlyDb, }); - const ponderApp = new Hono() - .use(cors()) - .get("/metrics", async (c) => { - try { - const metrics = await common.metrics.getMetrics(); - return c.text(metrics); - } catch (error) { - return c.json(error as Error, 500); - } - }) - .get("/health", async (c) => { - const status = await metadataStore.getStatus(); - - if ( - status !== null && - Object.values(status).every(({ ready }) => ready === true) - ) { - return c.text("", 200); - } - - const elapsed = (Date.now() - startTime) / 1000; - const max = common.options.maxHealthcheckDuration; - - if (elapsed > max) { - common.logger.warn({ - service: "server", - msg: `Historical indexing duration has exceeded the max healthcheck duration of ${max} seconds (current: ${elapsed}). Sevice is now responding as healthy and may serve incomplete data.`, - }); - return c.text("", 200); - } - - return c.text("Historical indexing is not complete.", 503); - }) - .get("/status", async (c) => { - const status = await metadataStore.getStatus(); - - return c.json(status); - }); - const metricsMiddleware = createMiddleware(async (c, next) => { const commonLabels = { method: c.req.method, path: c.req.path }; common.metrics.ponder_http_server_active_requests.inc(commonLabels); @@ -139,20 +100,46 @@ export async function createServer({ const hono = new Hono() .use(metricsMiddleware) - .route("/_ponder", ponderApp) - .use(contextMiddleware); + .use(cors()) + .get("/metrics", async (c) => { + try { + const metrics = await common.metrics.getMetrics(); + return c.text(metrics); + } catch (error) { + return c.json(error as Error, 500); + } + }) + .get("/health", async (c) => { + const status = await metadataStore.getStatus(); - if (userApp !== undefined && userRoutes !== undefined) { - for (const route of userApp.routes) { - // Validate user routes don't conflict with ponder routes - if (route.path.startsWith("/_ponder")) { + if ( + status !== null && + Object.values(status).every(({ ready }) => ready === true) + ) { + return c.text("", 200); + } + + const elapsed = (Date.now() - startTime) / 1000; + const max = common.options.maxHealthcheckDuration; + + if (elapsed > max) { common.logger.warn({ service: "server", - msg: `Ingoring '${route.method}' handler for route '${route.path}' because '/_ponder' is reserved for internal use`, + msg: `Historical indexing duration has exceeded the max healthcheck duration of ${max} seconds (current: ${elapsed}). Sevice is now responding as healthy and may serve incomplete data.`, }); + return c.text("", 200); } - } + return c.text("Historical indexing is not complete.", 503); + }) + .get("/status", async (c) => { + const status = await metadataStore.getStatus(); + + return c.json(status); + }) + .use(contextMiddleware); + + if (userApp !== undefined && userRoutes !== undefined) { const db = createDrizzleDb(database); const tables = convertSchemaToDrizzle(schema, database, dbNamespace); From 5e82d47e64e3afc05a2659340ccca50366b1d675 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Mon, 15 Jul 2024 18:28:24 -0400 Subject: [PATCH 100/122] drizzle list column --- packages/core/src/drizzle/list.ts | 100 ++++++++++++++++++ packages/core/src/drizzle/runtime.test.ts | 36 +++++++ packages/core/src/drizzle/runtime.ts | 122 +++++++++++++--------- packages/core/src/drizzle/table.test-d.ts | 28 +++-- 4 files changed, 232 insertions(+), 54 deletions(-) create mode 100644 packages/core/src/drizzle/list.ts diff --git a/packages/core/src/drizzle/list.ts b/packages/core/src/drizzle/list.ts new file mode 100644 index 000000000..2139c96ce --- /dev/null +++ b/packages/core/src/drizzle/list.ts @@ -0,0 +1,100 @@ +import type { Scalar } from "@/schema/common.js"; +import { entityKind } from "drizzle-orm"; +import { + type AnyPgTable, + PgColumn, + PgColumnBuilder, +} from "drizzle-orm/pg-core"; +import { + type AnySQLiteTable, + SQLiteColumn, + SQLiteColumnBuilder, +} from "drizzle-orm/sqlite-core"; + +export class PgListBuilder extends PgColumnBuilder { + static readonly [entityKind]: string = "PgListBuilder"; + element: Scalar; + + constructor(columnName: string, element: Scalar) { + super(columnName, "string", "PgList"); + this.element = element; + } + + build(table: AnyPgTable) { + return new PgList(table, this.config, this.element); + } +} + +export class PgList extends PgColumn { + static readonly [entityKind]: string = "PgList"; + element: Scalar; + + constructor( + table: AnyPgTable, + config: PgListBuilder["config"], + element: Scalar, + ) { + super(table, config); + this.element = element; + } + + getSQLType(): string { + return "text"; + } + + override mapFromDriverValue(value: string) { + return this.element === "bigint" + ? JSON.parse(value).map(BigInt) + : JSON.parse(value); + } + + override mapToDriverValue(value: Array): string { + return this.element === "bigint" + ? JSON.stringify(value.map(String)) + : JSON.stringify(value); + } +} + +export class SQLiteListBuilder extends SQLiteColumnBuilder { + static readonly [entityKind]: string = "SQliteListBuilder"; + element: Scalar; + + constructor(columnName: string, element: Scalar) { + super(columnName, "string", "PgList"); + this.element = element; + } + + build(table: AnySQLiteTable) { + return new SQLiteList(table, this.config, this.element); + } +} + +export class SQLiteList extends SQLiteColumn { + static readonly [entityKind]: string = "SQLiteList"; + element: Scalar; + + constructor( + table: AnyPgTable, + config: SQLiteListBuilder["config"], + element: Scalar, + ) { + super(table, config); + this.element = element; + } + + getSQLType(): string { + return "text"; + } + + override mapFromDriverValue(value: string) { + return this.element === "bigint" + ? JSON.parse(value).map(BigInt) + : JSON.parse(value); + } + + override mapToDriverValue(value: Array): string { + return this.element === "bigint" + ? JSON.stringify(value.map(String)) + : JSON.stringify(value); + } +} diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index 5cb7b559b..1e1d13c6b 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -174,6 +174,42 @@ test("select enum", async (context) => { await cleanup(); }); +test("select list", async (context) => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.string(), + list: p.string().list(), + }), + })); + + const { database, cleanup, indexingStore, namespaceInfo } = + await setupDatabaseServices(context, { schema }); + + await indexingStore.create({ + tableName: "table", + id: "1", + data: { + list: ["big", "dog"], + }, + }); + await (indexingStore as HistoricalStore).flush({ isFullFlush: true }); + + const db = createDrizzleDb(database) as unknown as DrizzleDb; + + const drizzleTables = convertSchemaToDrizzle( + schema, + database, + namespaceInfo.userNamespace, + ) as Context["tables"]; + + const rows = await db.select().from(drizzleTables.table); + + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: "1", list: ["big", "dog"] }); + + await cleanup(); +}); + test("select with join", async (context) => { const schema = createSchema((p) => ({ account: p.createTable({ diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index f8276dd17..ba744580b 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -1,8 +1,9 @@ import type { DatabaseService } from "@/database/service.js"; -import type { Schema } from "@/schema/common.js"; +import type { Scalar, Schema } from "@/schema/common.js"; import { isEnumColumn, isJSONColumn, + isListColumn, isMaterialColumn, isOptionalColumn, isReferenceColumn, @@ -29,6 +30,7 @@ import { import { SQLiteBigintBuilder } from "./bigint.js"; import { PgHexBuilder, SQLiteHexBuilder } from "./hex.js"; import { SQLiteJsonBuilder } from "./json.js"; +import { PgListBuilder, SQLiteListBuilder } from "./list.js"; export const createDrizzleDb = (database: DatabaseService) => { if (database.kind === "postgres") { @@ -72,53 +74,69 @@ export const convertSchemaToDrizzle = ( database.kind, ); } else if (isEnumColumn(column)) { - drizzleColumns[columnName] = convertEnumColumn( - columnName, - database.kind, - ); + if (isListColumn(column)) { + drizzleColumns[columnName] = convertListColumn( + columnName, + database.kind, + "string", + ); + } else { + drizzleColumns[columnName] = convertEnumColumn( + columnName, + database.kind, + ); + } } else if (isScalarColumn(column) || isReferenceColumn(column)) { - switch (column[" scalar"]) { - case "string": - drizzleColumns[columnName] = convertStringColumn( - columnName, - database.kind, - ); - break; - - case "int": - drizzleColumns[columnName] = convertIntColumn( - columnName, - database.kind, - ); - break; - - case "boolean": - drizzleColumns[columnName] = convertBooleanColumn( - columnName, - database.kind, - ); - break; - - case "float": - drizzleColumns[columnName] = convertFloatColumn( - columnName, - database.kind, - ); - break; - - case "hex": - drizzleColumns[columnName] = convertHexColumn( - columnName, - database.kind, - ); - break; - - case "bigint": - drizzleColumns[columnName] = convertBigintColumn( - columnName, - database.kind, - ); - break; + if (isListColumn(column)) { + drizzleColumns[columnName] = convertListColumn( + columnName, + database.kind, + column[" scalar"], + ); + } else { + switch (column[" scalar"]) { + case "string": + drizzleColumns[columnName] = convertStringColumn( + columnName, + database.kind, + ); + break; + + case "int": + drizzleColumns[columnName] = convertIntColumn( + columnName, + database.kind, + ); + break; + + case "boolean": + drizzleColumns[columnName] = convertBooleanColumn( + columnName, + database.kind, + ); + break; + + case "float": + drizzleColumns[columnName] = convertFloatColumn( + columnName, + database.kind, + ); + break; + + case "hex": + drizzleColumns[columnName] = convertHexColumn( + columnName, + database.kind, + ); + break; + + case "bigint": + drizzleColumns[columnName] = convertBigintColumn( + columnName, + database.kind, + ); + break; + } } // apply column constraints @@ -197,7 +215,15 @@ const convertBigintColumn = ( : PgNumeric(columnName, { precision: 78 }); }; -// TODO(kyle) list +const convertListColumn = ( + columnName: string, + kind: "sqlite" | "postgres", + element: Scalar, +) => { + return kind === "sqlite" + ? new SQLiteListBuilder(columnName, element) + : new PgListBuilder(columnName, element); +}; const convertJsonColumn = (columnName: string, kind: "sqlite" | "postgres") => { return kind === "sqlite" diff --git a/packages/core/src/drizzle/table.test-d.ts b/packages/core/src/drizzle/table.test-d.ts index 5298dc9d6..a30d10111 100644 --- a/packages/core/src/drizzle/table.test-d.ts +++ b/packages/core/src/drizzle/table.test-d.ts @@ -50,7 +50,6 @@ test("select enum", async () => { e: p.createEnum(["yes", "no"]), table: p.createTable({ id: p.string(), - name: p.int().optional(), e: p.enum("e"), }), })); @@ -64,14 +63,13 @@ test("select enum", async () => { const result = await ({} as DrizzleDb).select().from(table); // ^? - expectTypeOf<{ id: string; name: number | null; e: "yes" | "no" }[]>(result); + expectTypeOf<{ id: string; e: "yes" | "no" }[]>(result); }); test("select json", async () => { const schema = createSchema((p) => ({ table: p.createTable({ id: p.string(), - name: p.int().optional(), json: p.json<{ a: number; b: string }>(), }), })); @@ -85,9 +83,27 @@ test("select json", async () => { const result = await ({} as DrizzleDb).select().from(table); // ^? - expectTypeOf< - { id: string; name: number | null; json: { a: number; b: string } }[] - >(result); + expectTypeOf<{ id: string; json: { a: number; b: string } }[]>(result); +}); + +test("select list", async () => { + const schema = createSchema((p) => ({ + table: p.createTable({ + id: p.string(), + list: p.string().list(), + }), + })); + + const table = {} as ConvertToDrizzleTable< + "table", + (typeof schema)["table"]["table"], + typeof schema + >; + + const result = await ({} as DrizzleDb).select().from(table); + // ^? + + expectTypeOf<{ id: string; list: string[] }[]>(result); }); test("select join", async () => { From 411b63845e2f909d32833df46adf74a26a8f3e66 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 11:51:55 -0400 Subject: [PATCH 101/122] nits --- packages/core/src/_test/utils.ts | 2 +- packages/core/src/bin/commands/codegen.ts | 2 +- packages/core/src/bin/commands/dev.ts | 48 +++++++----- packages/core/src/bin/commands/serve.ts | 20 +++-- packages/core/src/bin/commands/start.ts | 20 +++-- packages/core/src/bin/utils/run.test.ts | 16 ++-- packages/core/src/bin/utils/run.ts | 8 +- packages/core/src/bin/utils/runServer.ts | 17 ++-- packages/core/src/build/index.ts | 10 ++- packages/core/src/build/service.ts | 87 +++++++++++---------- packages/core/src/common/codegen.ts | 6 +- packages/core/src/common/options.ts | 8 +- packages/core/src/common/telemetry.ts | 4 +- packages/core/src/drizzle/db.ts | 2 +- packages/core/src/drizzle/runtime.test.ts | 16 ++-- packages/core/src/drizzle/runtime.ts | 4 +- packages/core/src/drizzle/table.test-d.ts | 16 ++-- packages/core/src/drizzle/table.ts | 2 +- packages/core/src/graphql/index.ts | 2 + packages/core/src/hono/context.ts | 6 +- packages/core/src/indexing/addStackTrace.ts | 8 +- packages/core/src/server/service.ts | 4 +- 22 files changed, 172 insertions(+), 136 deletions(-) diff --git a/packages/core/src/_test/utils.ts b/packages/core/src/_test/utils.ts index 5cb9edb50..bcfdff32b 100644 --- a/packages/core/src/_test/utils.ts +++ b/packages/core/src/_test/utils.ts @@ -694,7 +694,7 @@ export async function waitForIndexedBlock( reject(new Error("Timed out while waiting for the indexed block.")); }, 5_000); const interval = setInterval(async () => { - const response = await fetch(`http://localhost:${port}/_ponder/status`); + const response = await fetch(`http://localhost:${port}/status`); if (response.status === 200) { const status = (await response.json()) as Status | null; const statusBlockNumber = status diff --git a/packages/core/src/bin/commands/codegen.ts b/packages/core/src/bin/commands/codegen.ts index d2de2144f..b61e2c7dd 100644 --- a/packages/core/src/bin/commands/codegen.ts +++ b/packages/core/src/bin/commands/codegen.ts @@ -57,7 +57,7 @@ export async function codegen({ cliOptions }: { cliOptions: CliOptions }) { properties: { cli_command: "codegen" }, }); - runCodegen({ common, graphQLSchema: buildResult.build.graphQLSchema }); + runCodegen({ common, graphqlSchema: buildResult.build.graphqlSchema }); logger.info({ service: "codegen", msg: "Wrote ponder-env.d.ts" }); logger.info({ service: "codegen", msg: "Wrote schema.graphql" }); diff --git a/packages/core/src/bin/commands/dev.ts b/packages/core/src/bin/commands/dev.ts index d97ef10c8..65517317c 100644 --- a/packages/core/src/bin/commands/dev.ts +++ b/packages/core/src/bin/commands/dev.ts @@ -1,7 +1,12 @@ import { existsSync } from "node:fs"; import path from "node:path"; -import { type BuildResult, createBuildService } from "@/build/index.js"; -import type { Build, BuildResultServer, BuildServer } from "@/build/service.js"; +import { + type ApiBuild, + type ApiBuildResult, + type IndexingBuild, + type IndexingBuildResult, + createBuildService, +} from "@/build/index.js"; import { createLogger } from "@/common/logger.js"; import { MetricsService } from "@/common/metrics.js"; import { buildOptions } from "@/common/options.js"; @@ -67,8 +72,8 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); - let cachedBuild: Build; - let cachedBuildServer: BuildServer | undefined; + let cachedIndexingBuild: IndexingBuild; + let cachedApiBuild: ApiBuild | undefined; // Note: an update to the "indexing" build triggers a reload of the server runtime. // This is to ensure that updates to either the config or schema are reflected in @@ -78,8 +83,8 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { concurrency: 1, worker: async ( result: - | ({ type: "indexing" } & BuildResult) - | ({ type: "server" } & BuildResultServer), + | ({ type: "indexing" } & IndexingBuildResult) + | ({ type: "server" } & ApiBuildResult), ) => { if (result.type === "indexing") { await cleanupReloadable(); @@ -89,7 +94,7 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { uiService.reset(); metrics.resetMetrics(); - cachedBuild = result.build; + cachedIndexingBuild = result.build; cleanupReloadable = await run({ common, @@ -105,8 +110,8 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { cleanupReloadableServer = await runServer({ common, - build: cachedBuild, - buildServer: cachedBuildServer, + indexingBuild: cachedIndexingBuild, + apiBuild: cachedApiBuild, }); } else { // This handles build failures and indexing errors on hot reload. @@ -117,12 +122,12 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { await cleanupReloadableServer(); if (result.status === "success") { - cachedBuildServer = result.build; + cachedApiBuild = result.build; cleanupReloadableServer = await runServer({ common, - build: cachedBuild, - buildServer: result.build, + indexingBuild: cachedIndexingBuild, + apiBuild: result.build, }); } else { // This handles build failures and indexing errors on hot reload. @@ -133,14 +138,14 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { }, }); - const initialResult = await buildService.start({ + const initialIndexingResult = await buildService.start({ watch: true, onBuild: (buildResult) => { buildQueue.clear(); buildQueue.add({ type: "indexing", ...buildResult }); }, }); - const initialResultServer = await buildService.startServer({ + const initialApiResult = await buildService.startServer({ watch: true, onBuild: (buildResult) => { buildQueue.clear(); @@ -149,8 +154,8 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { }); if ( - initialResult.status === "error" || - initialResultServer.status === "error" + initialIndexingResult.status === "error" || + initialApiResult.status === "error" ) { await shutdown({ reason: "Failed intial build", code: 1 }); return cleanup; @@ -158,13 +163,16 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { telemetry.record({ name: "lifecycle:session_start", - properties: { cli_command: "dev", ...buildPayload(initialResult.build) }, + properties: { + cli_command: "dev", + ...buildPayload(initialIndexingResult.build), + }, }); - cachedBuild = initialResult.build; - cachedBuildServer = initialResultServer.build; + cachedIndexingBuild = initialIndexingResult.build; + cachedApiBuild = initialApiResult.build; - buildQueue.add({ type: "indexing", ...initialResult }); + buildQueue.add({ type: "indexing", ...initialIndexingResult }); return async () => { buildQueue.pause(); diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index e3e9a6942..73c7db9ea 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -50,22 +50,28 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); - const buildResult = await buildService.start({ watch: false }); - const buildResultServer = await buildService.startServer({ watch: false }); + const indexingBuildResult = await buildService.start({ watch: false }); + const apiBuildResult = await buildService.startServer({ watch: false }); // Once we have the initial build, we can kill the build service. await buildService.kill(); - if (buildResult.status === "error" || buildResultServer.status === "error") { + if ( + indexingBuildResult.status === "error" || + apiBuildResult.status === "error" + ) { await shutdown({ reason: "Failed intial build", code: 1 }); return cleanup; } telemetry.record({ name: "lifecycle:session_start", - properties: { cli_command: "serve", ...buildPayload(buildResult.build) }, + properties: { + cli_command: "serve", + ...buildPayload(indexingBuildResult.build), + }, }); - const { databaseConfig, optionsConfig, schema } = buildResult.build; + const { databaseConfig, optionsConfig, schema } = indexingBuildResult.build; common.options = { ...common.options, ...optionsConfig }; @@ -96,8 +102,8 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { }); const server = await createServer({ - app: buildResultServer.build?.app, - routes: buildResultServer.build?.routes, + app: apiBuildResult.build?.app, + routes: apiBuildResult.build?.routes, common, schema, database, diff --git a/packages/core/src/bin/commands/start.ts b/packages/core/src/bin/commands/start.ts index 6ad6eac14..d46bd696c 100644 --- a/packages/core/src/bin/commands/start.ts +++ b/packages/core/src/bin/commands/start.ts @@ -52,24 +52,30 @@ export async function start({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); - const buildResult = await buildService.start({ watch: false }); - const buildResultServer = await buildService.startServer({ watch: false }); + const indexingBuildResult = await buildService.start({ watch: false }); + const apiBuildResult = await buildService.startServer({ watch: false }); // Once we have the initial build, we can kill the build service. await buildService.kill(); - if (buildResult.status === "error" || buildResultServer.status === "error") { + if ( + indexingBuildResult.status === "error" || + apiBuildResult.status === "error" + ) { await shutdown({ reason: "Failed intial build", code: 1 }); return cleanup; } telemetry.record({ name: "lifecycle:session_start", - properties: { cli_command: "start", ...buildPayload(buildResult.build) }, + properties: { + cli_command: "start", + ...buildPayload(indexingBuildResult.build), + }, }); cleanupReloadable = await run({ common, - build: buildResult.build, + build: indexingBuildResult.build, onFatalError: () => { shutdown({ reason: "Received fatal error", code: 1 }); }, @@ -80,8 +86,8 @@ export async function start({ cliOptions }: { cliOptions: CliOptions }) { cleanupReloadableServer = await runServer({ common, - build: buildResult.build, - buildServer: buildResultServer.build, + indexingBuild: indexingBuildResult.build, + apiBuild: apiBuildResult.build, }); return cleanup; diff --git a/packages/core/src/bin/utils/run.test.ts b/packages/core/src/bin/utils/run.test.ts index dc8a6fdaf..72e961421 100644 --- a/packages/core/src/bin/utils/run.test.ts +++ b/packages/core/src/bin/utils/run.test.ts @@ -3,7 +3,7 @@ import { setupCommon, setupIsolatedDatabase, } from "@/_test/setup.js"; -import type { Build } from "@/build/index.js"; +import type { IndexingBuild } from "@/build/index.js"; import * as codegen from "@/common/codegen.js"; import { buildGraphQLSchema } from "@/graphql/buildGraphqlSchema.js"; import { createSchema } from "@/schema/schema.js"; @@ -26,13 +26,13 @@ const schema = createSchema((p) => ({ }), })); -const graphQLSchema = buildGraphQLSchema(schema); +const graphqlSchema = buildGraphQLSchema(schema); test("run() kill", async (context) => { - const build: Build = { + const build: IndexingBuild = { buildId: "buildId", schema, - graphQLSchema, + graphqlSchema, databaseConfig: context.databaseConfig, optionsConfig: {}, networks: context.networks, @@ -59,10 +59,10 @@ test("run() setup", async (context) => { "Erc20:setup": vi.fn(), }; - const build: Build = { + const build: IndexingBuild = { buildId: "buildId", schema, - graphQLSchema, + graphqlSchema, databaseConfig: context.databaseConfig, optionsConfig: {}, networks: context.networks, @@ -88,10 +88,10 @@ test("run() setup error", async (context) => { }; const onReloadableErrorPromiseResolver = promiseWithResolvers(); - const build: Build = { + const build: IndexingBuild = { buildId: "buildId", schema, - graphQLSchema, + graphqlSchema, databaseConfig: context.databaseConfig, optionsConfig: {}, networks: context.networks, diff --git a/packages/core/src/bin/utils/run.ts b/packages/core/src/bin/utils/run.ts index 9c838a738..14245e2ca 100644 --- a/packages/core/src/bin/utils/run.ts +++ b/packages/core/src/bin/utils/run.ts @@ -1,4 +1,4 @@ -import type { Build } from "@/build/index.js"; +import type { IndexingBuild } from "@/build/index.js"; import { runCodegen } from "@/common/codegen.js"; import type { Common } from "@/common/common.js"; import { PostgresDatabaseService } from "@/database/postgres/service.js"; @@ -49,7 +49,7 @@ export async function run({ onReloadableError, }: { common: Common; - build: Build; + build: IndexingBuild; onFatalError: (error: Error) => void; onReloadableError: (error: Error) => void; }) { @@ -59,7 +59,7 @@ export async function run({ optionsConfig, networks, sources, - graphQLSchema, + graphqlSchema, schema, indexingFunctions, } = build; @@ -112,7 +112,7 @@ export async function run({ // starting the server so the app can become responsive more quickly. await database.migrateSyncStore(); - runCodegen({ common, graphQLSchema }); + runCodegen({ common, graphqlSchema }); // Note: can throw const syncService = await createSyncService({ diff --git a/packages/core/src/bin/utils/runServer.ts b/packages/core/src/bin/utils/runServer.ts index 88ae1eca2..7b8564595 100644 --- a/packages/core/src/bin/utils/runServer.ts +++ b/packages/core/src/bin/utils/runServer.ts @@ -1,5 +1,4 @@ -import type { Build } from "@/build/index.js"; -import type { BuildServer } from "@/build/service.js"; +import type { ApiBuild, IndexingBuild } from "@/build/index.js"; import type { Common } from "@/common/common.js"; import { PostgresDatabaseService } from "@/database/postgres/service.js"; import type { DatabaseService } from "@/database/service.js"; @@ -11,14 +10,14 @@ import { createServer } from "@/server/service.js"; */ export async function runServer({ common, - build, - buildServer, + indexingBuild, + apiBuild, }: { common: Common; - build: Build; - buildServer: BuildServer | undefined; + indexingBuild: IndexingBuild; + apiBuild: ApiBuild | undefined; }) { - const { databaseConfig, optionsConfig, schema } = build; + const { databaseConfig, optionsConfig, schema } = indexingBuild; common.options = { ...common.options, ...optionsConfig }; @@ -38,8 +37,8 @@ export async function runServer({ } const server = await createServer({ - app: buildServer?.app, - routes: buildServer?.routes, + app: apiBuild?.app, + routes: apiBuild?.routes, common, schema, database, diff --git a/packages/core/src/build/index.ts b/packages/core/src/build/index.ts index c36d6b6c8..8f5e98c1a 100644 --- a/packages/core/src/build/index.ts +++ b/packages/core/src/build/index.ts @@ -1,6 +1,12 @@ import { type Extend, extend } from "@/utils/extend.js"; import { create, kill, start, startServer } from "./service.js"; -import type { Build, BuildResult, Service } from "./service.js"; +import type { + ApiBuild, + ApiBuildResult, + IndexingBuild, + IndexingBuildResult, + Service, +} from "./service.js"; const methods = { start, startServer, kill }; @@ -8,4 +14,4 @@ export const createBuildService = extend(create, methods); export type BuildService = Extend; -export type { BuildResult, Build }; +export type { IndexingBuild, IndexingBuildResult, ApiBuild, ApiBuildResult }; diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index f16ff7541..eb5628497 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -33,10 +33,10 @@ const BUILD_ID_VERSION = "1"; export type Service = { // static common: Common; - srcRegex: RegExp; - serverRegex: RegExp; - srcPattern: string; - serverPattern: string; + indexingRegex: RegExp; + apiRegex: RegExp; + indexingPattern: string; + apiPattern: string; // vite viteDevServer: ViteDevServer; @@ -44,7 +44,7 @@ export type Service = { viteNodeRunner: ViteNodeRunner; }; -export type Build = { +export type IndexingBuild = { // Build ID for caching buildId: string; // Config @@ -54,22 +54,22 @@ export type Build = { networks: Network[]; // Schema schema: Schema; - graphQLSchema: GraphQLSchema; + graphqlSchema: GraphQLSchema; // Indexing functions indexingFunctions: IndexingFunctions; }; -export type BuildServer = { +export type ApiBuild = { app: Hono; routes: PonderRoutes; }; -export type BuildResult = - | { status: "success"; build: Build } +export type IndexingBuildResult = + | { status: "success"; build: IndexingBuild } | { status: "error"; error: Error }; -export type BuildResultServer = - | { status: "success"; build?: BuildServer } +export type ApiBuildResult = + | { status: "success"; build?: ApiBuild } | { status: "error"; error: Error }; type RawBuild = { @@ -88,26 +88,26 @@ export const create = async ({ }): Promise => { const escapeRegex = /[.*+?^${}()|[\]\\]/g; - const escapedSrcDir = common.options.srcDir + const escapedIndexingDir = common.options.indexingDir // If on Windows, use a POSIX path for this regex. .replace(/\\/g, "/") // Escape special characters in the path. .replace(escapeRegex, "\\$&"); - const srcRegex = new RegExp(`^${escapedSrcDir}/.*\\.(ts|js)$`); + const indexingRegex = new RegExp(`^${escapedIndexingDir}/.*\\.(ts|js)$`); - const escapedServerDir = common.options.serverDir + const escapedApiDir = common.options.apiDir // If on Windows, use a POSIX path for this regex. .replace(/\\/g, "/") // Escape special characters in the path. .replace(escapeRegex, "\\$&"); - const serverRegex = new RegExp(`^${escapedServerDir}/.*\\.(ts|js)$`); + const apiRegex = new RegExp(`^${escapedApiDir}/.*\\.(ts|js)$`); - const srcPattern = path - .join(common.options.srcDir, "**/*.{js,mjs,ts,mts}") + const indexingPattern = path + .join(common.options.indexingDir, "**/*.{js,mjs,ts,mts}") .replace(/\\/g, "/"); - const serverPattern = path - .join(common.options.serverDir, "**/*.{js,mjs,ts,mts}") + const apiPattern = path + .join(common.options.apiDir, "**/*.{js,mjs,ts,mts}") .replace(/\\/g, "/"); const viteLogger = { @@ -160,10 +160,10 @@ export const create = async ({ return { common, - srcRegex, - serverRegex, - srcPattern, - serverPattern, + indexingRegex, + apiRegex, + indexingPattern, + apiPattern, viteDevServer, viteNodeServer, viteNodeRunner, @@ -182,9 +182,9 @@ export const start = async ( watch, onBuild, }: - | { watch: true; onBuild: (buildResult: BuildResult) => void } + | { watch: true; onBuild: (buildResult: IndexingBuildResult) => void } | { watch: false; onBuild?: never }, -): Promise => { +): Promise => { const { common } = buildService; // Note: Don't run these in parallel. If there are circular imports in user code, @@ -260,8 +260,8 @@ export const start = async ( ); const hasSrcUpdate = invalidated.some( (file) => - buildService.srcRegex.test(file) && - !buildService.serverRegex.test(file), + buildService.indexingRegex.test(file) && + !buildService.apiRegex.test(file), ); // This branch could trigger if you change a `note.txt` file within `src/`. @@ -296,8 +296,8 @@ export const start = async ( } if (hasSrcUpdate) { - const files = glob.sync(buildService.srcPattern, { - ignore: buildService.serverPattern, + const files = glob.sync(buildService.indexingPattern, { + ignore: buildService.apiPattern, }); buildService.viteNodeRunner.moduleCache.invalidateDepTree(files); buildService.viteNodeRunner.moduleCache.deleteByModuleId("@/generated"); @@ -326,9 +326,9 @@ export const startServer = async ( watch, onBuild, }: - | { watch: true; onBuild: (buildResult: BuildResultServer) => void } + | { watch: true; onBuild: (buildResult: ApiBuildResult) => void } | { watch: false; onBuild?: never }, -): Promise => { +): Promise => { const { common } = buildService; const serverResult = await executeServer(buildService); @@ -362,7 +362,7 @@ export const startServer = async ( const onFileChange = async (file: string) => { if (isFileIgnored(file)) return; - const files = glob.sync(buildService.serverPattern); + const files = glob.sync(buildService.apiPattern); if (files.includes(file) === false) return; @@ -470,8 +470,8 @@ const executeIndexingFunctions = async ( } | { status: "error"; error: Error } > => { - const files = glob.sync(buildService.srcPattern, { - ignore: buildService.serverPattern, + const files = glob.sync(buildService.indexingPattern, { + ignore: buildService.apiPattern, }); const executeResults = await Promise.all( files.map(async (file) => ({ @@ -531,13 +531,16 @@ const executeServer = async ( } | { status: "error"; error: Error } > => { - const doesServerExist = fs.existsSync(buildService.common.options.serverDir); + const doesServerExist = fs.existsSync(buildService.common.options.apiDir); + const hasTsFileInServer = fs + .readdirSync(buildService.common.options.apiDir) + .some((file) => file.endsWith(".ts")); - if (doesServerExist === false) { + if (doesServerExist === false || hasTsFileInServer === false) { return { status: "success" }; } - const files = glob.sync(buildService.serverPattern); + const files = glob.sync(buildService.apiPattern); const executeResults = await Promise.all( files.map(async (file) => ({ ...(await executeFile(buildService, { file })), @@ -572,7 +575,7 @@ const executeServer = async ( const validateAndBuild = async ( { common }: Pick, rawBuild: RawBuild, -): Promise => { +): Promise => { // Validate and build the schema const buildSchemaResult = safeBuildSchema({ schema: rawBuild.schema.schema, @@ -591,7 +594,7 @@ const validateAndBuild = async ( common.logger[log.level]({ service: "build", msg: log.msg }); } - const graphQLSchema = buildGraphQLSchema(buildSchemaResult.schema); + const graphqlSchema = buildGraphQLSchema(buildSchemaResult.schema); // Validates and build the config const buildConfigAndIndexingFunctionsResult = @@ -636,7 +639,7 @@ const validateAndBuild = async ( networks: buildConfigAndIndexingFunctionsResult.networks, sources: buildConfigAndIndexingFunctionsResult.sources, schema: buildSchemaResult.schema, - graphQLSchema, + graphqlSchema, indexingFunctions: buildConfigAndIndexingFunctionsResult.indexingFunctions, }, @@ -645,8 +648,8 @@ const validateAndBuild = async ( const validateAndBuildServer = ( { common }: Pick, - build: Partial, -): BuildResultServer => { + build: Partial, +): ApiBuildResult => { if (!build.app || !build.routes) { return { status: "success" }; } diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index 6fe66b47a..72848ecdf 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -34,10 +34,10 @@ declare module "@/generated" { export function runCodegen({ common, - graphQLSchema, + graphqlSchema, }: { common: Common; - graphQLSchema: GraphQLSchema; + graphqlSchema: GraphQLSchema; }) { writeFileSync( path.join(common.options.rootDir, "ponder-env.d.ts"), @@ -53,7 +53,7 @@ export function runCodegen({ mkdirSync(common.options.generatedDir, { recursive: true }); writeFileSync( path.join(common.options.generatedDir, "schema.graphql"), - printSchema(graphQLSchema), + printSchema(graphqlSchema), "utf-8", ); diff --git a/packages/core/src/common/options.ts b/packages/core/src/common/options.ts index 7e4db1c8f..8fcb23123 100644 --- a/packages/core/src/common/options.ts +++ b/packages/core/src/common/options.ts @@ -9,8 +9,8 @@ export type Options = { configFile: string; schemaFile: string; rootDir: string; - srcDir: string; - serverDir: string; + indexingDir: string; + apiDir: string; generatedDir: string; ponderDir: string; logDir: string; @@ -79,8 +79,8 @@ export const buildOptions = ({ cliOptions }: { cliOptions: CliOptions }) => { rootDir, configFile: path.join(rootDir, cliOptions.config), schemaFile: path.join(rootDir, "ponder.schema.ts"), - srcDir: path.join(rootDir, "src"), - serverDir: path.join(rootDir, "src", "api"), + indexingDir: path.join(rootDir, "src"), + apiDir: path.join(rootDir, "src", "api"), generatedDir: path.join(rootDir, "generated"), ponderDir: path.join(rootDir, ".ponder"), logDir: path.join(rootDir, ".ponder", "logs"), diff --git a/packages/core/src/common/telemetry.ts b/packages/core/src/common/telemetry.ts index 671e0389f..913f05141 100644 --- a/packages/core/src/common/telemetry.ts +++ b/packages/core/src/common/telemetry.ts @@ -4,7 +4,7 @@ import { existsSync, readFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; -import type { Build } from "@/build/service.js"; +import type { IndexingBuild } from "@/build/service.js"; import type { Options } from "@/common/options.js"; import { getTables } from "@/schema/utils.js"; import { startClock } from "@/utils/timer.js"; @@ -269,7 +269,7 @@ function getPackageJson(rootDir: string) { } } -export function buildPayload(build: Build) { +export function buildPayload(build: IndexingBuild) { const table_count = Object.keys(getTables(build.schema)).length; const indexing_function_count = Object.values(build.indexingFunctions).reduce( (acc, f) => acc + Object.keys(f).length, diff --git a/packages/core/src/drizzle/db.ts b/packages/core/src/drizzle/db.ts index d12fe7fd6..76b87d550 100644 --- a/packages/core/src/drizzle/db.ts +++ b/packages/core/src/drizzle/db.ts @@ -10,7 +10,7 @@ export type DrizzleDb = { fields?: SelectedFields, ): SelectBuilder | undefined, "async", void>; /** - * Execute raw, readonly sql queries. + * Execute a raw read-only SQL query.. * * @example * import { ponder } from "@/generated"; diff --git a/packages/core/src/drizzle/runtime.test.ts b/packages/core/src/drizzle/runtime.test.ts index 1e1d13c6b..3e8978947 100644 --- a/packages/core/src/drizzle/runtime.test.ts +++ b/packages/core/src/drizzle/runtime.test.ts @@ -9,7 +9,7 @@ import { createSchema } from "@/schema/schema.js"; import { eq } from "drizzle-orm"; import { beforeEach, expect, test } from "vitest"; import type { DrizzleDb } from "./db.js"; -import { convertSchemaToDrizzle, createDrizzleDb } from "./runtime.js"; +import { createDrizzleDb, createDrizzleTables } from "./runtime.js"; beforeEach(setupCommon); beforeEach(setupIsolatedDatabase); @@ -29,7 +29,7 @@ test("runtime select", async (context) => { const db = createDrizzleDb(database) as unknown as DrizzleDb; - const drizzleTables = convertSchemaToDrizzle( + const drizzleTables = createDrizzleTables( schema, database, namespaceInfo.userNamespace, @@ -58,7 +58,7 @@ test("select hex", async (context) => { const db = createDrizzleDb(database) as unknown as DrizzleDb; - const drizzleTables = convertSchemaToDrizzle( + const drizzleTables = createDrizzleTables( schema, database, namespaceInfo.userNamespace, @@ -87,7 +87,7 @@ test("select bigint", async (context) => { const db = createDrizzleDb(database) as unknown as DrizzleDb; - const drizzleTables = convertSchemaToDrizzle( + const drizzleTables = createDrizzleTables( schema, database, namespaceInfo.userNamespace, @@ -125,7 +125,7 @@ test("select json", async (context) => { const db = createDrizzleDb(database) as unknown as DrizzleDb; - const drizzleTables = convertSchemaToDrizzle( + const drizzleTables = createDrizzleTables( schema, database, namespaceInfo.userNamespace, @@ -160,7 +160,7 @@ test("select enum", async (context) => { const db = createDrizzleDb(database) as unknown as DrizzleDb; - const drizzleTables = convertSchemaToDrizzle( + const drizzleTables = createDrizzleTables( schema, database, namespaceInfo.userNamespace, @@ -196,7 +196,7 @@ test("select list", async (context) => { const db = createDrizzleDb(database) as unknown as DrizzleDb; - const drizzleTables = convertSchemaToDrizzle( + const drizzleTables = createDrizzleTables( schema, database, namespaceInfo.userNamespace, @@ -243,7 +243,7 @@ test("select with join", async (context) => { const db = createDrizzleDb(database) as unknown as DrizzleDb; - const drizzleTables = convertSchemaToDrizzle( + const drizzleTables = createDrizzleTables( schema, database, namespaceInfo.userNamespace, diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index ba744580b..763949a51 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -56,7 +56,7 @@ type SQLiteTable = Parameters[1]; type PostgresTable = Parameters[1]; type DrizzleTable = { [tableName: string]: any }; -export const convertSchemaToDrizzle = ( +export const createDrizzleTables = ( schema: Schema, database: DatabaseService, dbNamespace: string, @@ -151,6 +151,8 @@ export const convertSchemaToDrizzle = ( } if (database.kind === "postgres") { + // Note: this is to avoid an error thrown by drizzle when + // setting schema to "public". if (dbNamespace === "public") { drizzleTables[tableName] = pgTable( tableName, diff --git a/packages/core/src/drizzle/table.test-d.ts b/packages/core/src/drizzle/table.test-d.ts index a30d10111..7c57393be 100644 --- a/packages/core/src/drizzle/table.test-d.ts +++ b/packages/core/src/drizzle/table.test-d.ts @@ -3,7 +3,7 @@ import { eq } from "drizzle-orm"; import type { Hex } from "viem"; import { expectTypeOf, test } from "vitest"; import type { DrizzleDb } from "./db.js"; -import type { ConvertToDrizzleTable } from "./table.js"; +import type { DrizzleTable } from "./table.js"; test("select query promise", async () => { const schema = createSchema((p) => ({ @@ -13,7 +13,7 @@ test("select query promise", async () => { }), })); - const table = {} as ConvertToDrizzleTable< + const table = {} as DrizzleTable< "table", (typeof schema)["table"]["table"], typeof schema @@ -33,7 +33,7 @@ test("select optional column", async () => { }), })); - const table = {} as ConvertToDrizzleTable< + const table = {} as DrizzleTable< "table", (typeof schema)["table"]["table"], typeof schema @@ -54,7 +54,7 @@ test("select enum", async () => { }), })); - const table = {} as ConvertToDrizzleTable< + const table = {} as DrizzleTable< "table", (typeof schema)["table"]["table"], typeof schema @@ -74,7 +74,7 @@ test("select json", async () => { }), })); - const table = {} as ConvertToDrizzleTable< + const table = {} as DrizzleTable< "table", (typeof schema)["table"]["table"], typeof schema @@ -94,7 +94,7 @@ test("select list", async () => { }), })); - const table = {} as ConvertToDrizzleTable< + const table = {} as DrizzleTable< "table", (typeof schema)["table"]["table"], typeof schema @@ -119,12 +119,12 @@ test("select join", async () => { }), })); - const account = {} as ConvertToDrizzleTable< + const account = {} as DrizzleTable< "account", (typeof schema)["account"]["table"], typeof schema >; - const nft = {} as ConvertToDrizzleTable< + const nft = {} as DrizzleTable< "nft", (typeof schema)["nft"]["table"], typeof schema diff --git a/packages/core/src/drizzle/table.ts b/packages/core/src/drizzle/table.ts index 4ee46d48d..6fe78fc6a 100644 --- a/packages/core/src/drizzle/table.ts +++ b/packages/core/src/drizzle/table.ts @@ -16,7 +16,7 @@ import type { TableWithColumns } from "./select.js"; * * @returns TableWithColumns */ -export type ConvertToDrizzleTable< +export type DrizzleTable< tableName extends string, table extends PonderTable, schema extends PonderSchema, diff --git a/packages/core/src/graphql/index.ts b/packages/core/src/graphql/index.ts index 009246931..12f0f32fd 100644 --- a/packages/core/src/graphql/index.ts +++ b/packages/core/src/graphql/index.ts @@ -14,6 +14,8 @@ import { buildLoaderCache } from "./buildLoaderCache.js"; * * @example * import { ponder } from "@/generated"; + * import { graphql } from "@ponder/core"; + * * ponder.use("/graphql", graphql()); * */ diff --git a/packages/core/src/hono/context.ts b/packages/core/src/hono/context.ts index 65901ea5a..8fe3a0a3d 100644 --- a/packages/core/src/hono/context.ts +++ b/packages/core/src/hono/context.ts @@ -1,5 +1,5 @@ import type { DrizzleDb } from "@/drizzle/db.js"; -import type { ConvertToDrizzleTable } from "@/drizzle/table.js"; +import type { DrizzleTable } from "@/drizzle/table.js"; import type { ExtractTableNames, Schema } from "@/schema/common.js"; import type { Env, Context as HonoContext, Input } from "hono"; @@ -10,7 +10,7 @@ export type Context< > = { db: DrizzleDb; tables: { - [tableName in ExtractTableNames]: ConvertToDrizzleTable< + [tableName in ExtractTableNames]: DrizzleTable< tableName, // @ts-ignore schema[tableName]["table"], @@ -62,7 +62,7 @@ export type MiddlewareContext< > = HonoContext & { db: DrizzleDb; tables: { - [tableName in ExtractTableNames]: ConvertToDrizzleTable< + [tableName in ExtractTableNames]: DrizzleTable< tableName, // @ts-ignore schema[tableName]["table"], diff --git a/packages/core/src/indexing/addStackTrace.ts b/packages/core/src/indexing/addStackTrace.ts index fd10a80d9..84bed7a45 100644 --- a/packages/core/src/indexing/addStackTrace.ts +++ b/packages/core/src/indexing/addStackTrace.ts @@ -3,6 +3,10 @@ import type { Options } from "@/common/options.js"; import { codeFrameColumns } from "@babel/code-frame"; import { type StackFrame, parse as parseStackTrace } from "stacktrace-parser"; +// Note: this currently works for both indexing functions and api +// routes only because the api route dir is a subdir of the indexing function +// dir. + export const addStackTrace = (error: Error, options: Options) => { if (!error.stack) return; @@ -13,12 +17,12 @@ export const addStackTrace = (error: Error, options: Options) => { // Find first frame that occurred within user code. const firstUserFrameIndex = stackTrace.findIndex((frame) => - frame.file?.includes(options.srcDir), + frame.file?.includes(options.indexingDir), ); if (firstUserFrameIndex >= 0) { userStackTrace = stackTrace.filter((frame) => - frame.file?.includes(options.srcDir), + frame.file?.includes(options.indexingDir), ); const firstUserFrame = stackTrace[firstUserFrameIndex]; diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 3df8f1d6f..885085fe8 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -1,7 +1,7 @@ import http from "node:http"; import type { Common } from "@/common/common.js"; import type { DatabaseService } from "@/database/service.js"; -import { convertSchemaToDrizzle, createDrizzleDb } from "@/drizzle/runtime.js"; +import { createDrizzleDb, createDrizzleTables } from "@/drizzle/runtime.js"; import { graphql } from "@/graphql/index.js"; import { type PonderRoutes, applyHonoRoutes } from "@/hono/index.js"; import { getMetadataStore } from "@/indexing-store/metadata.js"; @@ -141,7 +141,7 @@ export async function createServer({ if (userApp !== undefined && userRoutes !== undefined) { const db = createDrizzleDb(database); - const tables = convertSchemaToDrizzle(schema, database, dbNamespace); + const tables = createDrizzleTables(schema, database, dbNamespace); // apply user routes to hono instance, registering a custom error handler applyHonoRoutes(hono, userRoutes, { db, tables }).onError((error, c) => From 8672adff0a52492ca80214fdf4f0044ff8bd4ba6 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 13:06:20 -0400 Subject: [PATCH 102/122] fix graphql middleware --- packages/core/src/graphql/index.ts | 63 ++++++++++++++++-------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/packages/core/src/graphql/index.ts b/packages/core/src/graphql/index.ts index 12f0f32fd..48e5bcfbb 100644 --- a/packages/core/src/graphql/index.ts +++ b/packages/core/src/graphql/index.ts @@ -35,40 +35,45 @@ export const graphql = ( maxOperationDepth: 100, maxOperationAliases: 30, }, -) => - createMiddleware(async (c) => { - const readonlyStore = c.get("readonlyStore"); - const metadataStore = c.get("metadataStore"); - const schema = c.get("schema"); - const graphqlSchema = buildGraphQLSchema(schema); +) => { + let yoga: any | undefined = undefined; + return createMiddleware(async (c) => { if (c.req.method === "GET") { return c.html(graphiQLHtml(c.req.path)); } - const yoga = createYoga({ - schema: graphqlSchema, - context: () => { - const getLoader = buildLoaderCache({ store: readonlyStore }); - return { readonlyStore, metadataStore, getLoader }; - }, - graphqlEndpoint: c.req.path, - maskedErrors: process.env.NODE_ENV === "production", - logging: false, - graphiql: false, - parserAndValidationCache: false, - plugins: [ - maxTokensPlugin({ n: maxOperationTokens }), - maxDepthPlugin({ - n: maxOperationDepth, - ignoreIntrospection: false, - }), - maxAliasesPlugin({ - n: maxOperationAliases, - allowList: [], - }), - ], - }); + if (yoga === undefined) { + const readonlyStore = c.get("readonlyStore"); + const metadataStore = c.get("metadataStore"); + const schema = c.get("schema"); + const graphqlSchema = buildGraphQLSchema(schema); + + yoga = createYoga({ + schema: graphqlSchema, + context: () => { + const getLoader = buildLoaderCache({ store: readonlyStore }); + return { readonlyStore, metadataStore, getLoader }; + }, + graphqlEndpoint: c.req.path, + maskedErrors: process.env.NODE_ENV === "production", + logging: false, + graphiql: false, + parserAndValidationCache: false, + plugins: [ + maxTokensPlugin({ n: maxOperationTokens }), + maxDepthPlugin({ + n: maxOperationDepth, + ignoreIntrospection: false, + }), + maxAliasesPlugin({ + n: maxOperationAliases, + allowList: [], + }), + ], + }); + } return yoga.handle(c.req.raw); }); +}; From 7ea2e350e2bb53e6bca976d98abb22d48c3fe7b1 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 14:37:02 -0400 Subject: [PATCH 103/122] refactor build service to emit two separate events --- packages/core/src/bin/commands/codegen.ts | 8 +- packages/core/src/bin/commands/dev.ts | 144 +++++------ packages/core/src/bin/commands/serve.ts | 16 +- packages/core/src/bin/commands/start.ts | 15 +- packages/core/src/bin/utils/runServer.ts | 14 +- packages/core/src/build/index.ts | 4 +- packages/core/src/build/service.ts | 292 ++++++++++++---------- 7 files changed, 249 insertions(+), 244 deletions(-) diff --git a/packages/core/src/bin/commands/codegen.ts b/packages/core/src/bin/commands/codegen.ts index b61e2c7dd..ad49b0864 100644 --- a/packages/core/src/bin/commands/codegen.ts +++ b/packages/core/src/bin/commands/codegen.ts @@ -40,13 +40,13 @@ export async function codegen({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); - const buildResult = await buildService.start({ watch: false }); + const { indexing } = await buildService.start({ watch: false }); - if (buildResult.status === "error") { + if (indexing.status === "error") { logger.error({ service: "process", msg: "Failed schema build", - error: buildResult.error, + error: indexing.error, }); await shutdown({ reason: "Failed schema build", code: 1 }); return; @@ -57,7 +57,7 @@ export async function codegen({ cliOptions }: { cliOptions: CliOptions }) { properties: { cli_command: "codegen" }, }); - runCodegen({ common, graphqlSchema: buildResult.build.graphqlSchema }); + runCodegen({ common, graphqlSchema: indexing.build.graphqlSchema }); logger.info({ service: "codegen", msg: "Wrote ponder-env.d.ts" }); logger.info({ service: "codegen", msg: "Wrote schema.graphql" }); diff --git a/packages/core/src/bin/commands/dev.ts b/packages/core/src/bin/commands/dev.ts index 65517317c..21e5f23b4 100644 --- a/packages/core/src/bin/commands/dev.ts +++ b/packages/core/src/bin/commands/dev.ts @@ -1,9 +1,7 @@ import { existsSync } from "node:fs"; import path from "node:path"; import { - type ApiBuild, type ApiBuildResult, - type IndexingBuild, type IndexingBuildResult, createBuildService, } from "@/build/index.js"; @@ -59,12 +57,12 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { const uiService = new UiService({ common }); - let cleanupReloadable = () => Promise.resolve(); - let cleanupReloadableServer = () => Promise.resolve(); + let indexingCleanupReloadable = () => Promise.resolve(); + let apiCleanupReloadable = () => Promise.resolve(); const cleanup = async () => { - await cleanupReloadable(); - await cleanupReloadableServer(); + await indexingCleanupReloadable(); + await apiCleanupReloadable(); await buildService.kill(); await telemetry.kill(); uiService.kill(); @@ -72,91 +70,70 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); - let cachedIndexingBuild: IndexingBuild; - let cachedApiBuild: ApiBuild | undefined; - - // Note: an update to the "indexing" build triggers a reload of the server runtime. - // This is to ensure that updates to either the config or schema are reflected in - // the server. - const buildQueue = createQueue({ + const indexingBuildQueue = createQueue({ initialStart: true, concurrency: 1, - worker: async ( - result: - | ({ type: "indexing" } & IndexingBuildResult) - | ({ type: "server" } & ApiBuildResult), - ) => { - if (result.type === "indexing") { - await cleanupReloadable(); - await cleanupReloadableServer(); - - if (result.status === "success") { - uiService.reset(); - metrics.resetMetrics(); - - cachedIndexingBuild = result.build; - - cleanupReloadable = await run({ - common, - build: result.build, - onFatalError: () => { - shutdown({ reason: "Received fatal error", code: 1 }); - }, - onReloadableError: (error) => { - buildQueue.clear(); - buildQueue.add({ type: "indexing", status: "error", error }); - }, - }); - - cleanupReloadableServer = await runServer({ - common, - indexingBuild: cachedIndexingBuild, - apiBuild: cachedApiBuild, - }); - } else { - // This handles build failures and indexing errors on hot reload. - uiService.setReloadableError(); - cleanupReloadable = () => Promise.resolve(); - } + worker: async (result: IndexingBuildResult) => { + await indexingCleanupReloadable(); + + if (result.status === "success") { + uiService.reset(); + metrics.resetMetrics(); + + indexingCleanupReloadable = await run({ + common, + build: result.build, + onFatalError: () => { + shutdown({ reason: "Received fatal error", code: 1 }); + }, + onReloadableError: (error) => { + indexingBuildQueue.clear(); + indexingBuildQueue.add({ status: "error", error }); + }, + }); } else { - await cleanupReloadableServer(); - - if (result.status === "success") { - cachedApiBuild = result.build; - - cleanupReloadableServer = await runServer({ - common, - indexingBuild: cachedIndexingBuild, - apiBuild: result.build, - }); - } else { - // This handles build failures and indexing errors on hot reload. - uiService.setReloadableError(); - cleanupReloadableServer = () => Promise.resolve(); - } + // This handles build failures and indexing errors on hot reload. + uiService.setReloadableError(); + indexingCleanupReloadable = () => Promise.resolve(); } }, }); - const initialIndexingResult = await buildService.start({ - watch: true, - onBuild: (buildResult) => { - buildQueue.clear(); - buildQueue.add({ type: "indexing", ...buildResult }); + const apiBuildQueue = createQueue({ + initialStart: true, + concurrency: 1, + worker: async (result: ApiBuildResult) => { + await apiCleanupReloadable(); + + if (result.status === "success") { + uiService.reset(); + metrics.resetMetrics(); + + apiCleanupReloadable = await runServer({ + common, + build: result.build, + }); + } else { + // This handles build failures on hot reload. + uiService.setReloadableError(); + apiCleanupReloadable = () => Promise.resolve(); + } }, }); - const initialApiResult = await buildService.startServer({ + + const { api, indexing } = await buildService.start({ watch: true, - onBuild: (buildResult) => { - buildQueue.clear(); - buildQueue.add({ type: "server", ...buildResult }); + onIndexingBuild: (buildResult) => { + indexingBuildQueue.clear(); + indexingBuildQueue.add(buildResult); + }, + onApiBuild: (buildResult) => { + apiBuildQueue.clear(); + apiBuildQueue.add(buildResult); }, }); - if ( - initialIndexingResult.status === "error" || - initialApiResult.status === "error" - ) { + if (indexing.status === "error" || api.status === "error") { await shutdown({ reason: "Failed intial build", code: 1 }); return cleanup; } @@ -165,17 +142,16 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { name: "lifecycle:session_start", properties: { cli_command: "dev", - ...buildPayload(initialIndexingResult.build), + ...buildPayload(indexing.build), }, }); - cachedIndexingBuild = initialIndexingResult.build; - cachedApiBuild = initialApiResult.build; - - buildQueue.add({ type: "indexing", ...initialIndexingResult }); + indexingBuildQueue.add(indexing); + apiBuildQueue.add(api); return async () => { - buildQueue.pause(); + indexingBuildQueue.pause(); + apiBuildQueue.pause(); await cleanup(); }; } diff --git a/packages/core/src/bin/commands/serve.ts b/packages/core/src/bin/commands/serve.ts index 73c7db9ea..baf8d2d8c 100644 --- a/packages/core/src/bin/commands/serve.ts +++ b/packages/core/src/bin/commands/serve.ts @@ -50,15 +50,11 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); - const indexingBuildResult = await buildService.start({ watch: false }); - const apiBuildResult = await buildService.startServer({ watch: false }); + const { api, indexing } = await buildService.start({ watch: false }); // Once we have the initial build, we can kill the build service. await buildService.kill(); - if ( - indexingBuildResult.status === "error" || - apiBuildResult.status === "error" - ) { + if (api.status === "error" || indexing.status === "error") { await shutdown({ reason: "Failed intial build", code: 1 }); return cleanup; } @@ -67,11 +63,11 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { name: "lifecycle:session_start", properties: { cli_command: "serve", - ...buildPayload(indexingBuildResult.build), + ...buildPayload(indexing.build), }, }); - const { databaseConfig, optionsConfig, schema } = indexingBuildResult.build; + const { databaseConfig, optionsConfig, schema } = api.build; common.options = { ...common.options, ...optionsConfig }; @@ -102,8 +98,8 @@ export async function serve({ cliOptions }: { cliOptions: CliOptions }) { }); const server = await createServer({ - app: apiBuildResult.build?.app, - routes: apiBuildResult.build?.routes, + app: api.build.app, + routes: api.build.routes, common, schema, database, diff --git a/packages/core/src/bin/commands/start.ts b/packages/core/src/bin/commands/start.ts index d46bd696c..110698255 100644 --- a/packages/core/src/bin/commands/start.ts +++ b/packages/core/src/bin/commands/start.ts @@ -52,15 +52,11 @@ export async function start({ cliOptions }: { cliOptions: CliOptions }) { const shutdown = setupShutdown({ common, cleanup }); - const indexingBuildResult = await buildService.start({ watch: false }); - const apiBuildResult = await buildService.startServer({ watch: false }); + const { indexing, api } = await buildService.start({ watch: false }); // Once we have the initial build, we can kill the build service. await buildService.kill(); - if ( - indexingBuildResult.status === "error" || - apiBuildResult.status === "error" - ) { + if (indexing.status === "error" || api.status === "error") { await shutdown({ reason: "Failed intial build", code: 1 }); return cleanup; } @@ -69,13 +65,13 @@ export async function start({ cliOptions }: { cliOptions: CliOptions }) { name: "lifecycle:session_start", properties: { cli_command: "start", - ...buildPayload(indexingBuildResult.build), + ...buildPayload(indexing.build), }, }); cleanupReloadable = await run({ common, - build: indexingBuildResult.build, + build: indexing.build, onFatalError: () => { shutdown({ reason: "Received fatal error", code: 1 }); }, @@ -86,8 +82,7 @@ export async function start({ cliOptions }: { cliOptions: CliOptions }) { cleanupReloadableServer = await runServer({ common, - indexingBuild: indexingBuildResult.build, - apiBuild: apiBuildResult.build, + build: api.build, }); return cleanup; diff --git a/packages/core/src/bin/utils/runServer.ts b/packages/core/src/bin/utils/runServer.ts index 7b8564595..b2f62f159 100644 --- a/packages/core/src/bin/utils/runServer.ts +++ b/packages/core/src/bin/utils/runServer.ts @@ -1,4 +1,4 @@ -import type { ApiBuild, IndexingBuild } from "@/build/index.js"; +import type { ApiBuild } from "@/build/index.js"; import type { Common } from "@/common/common.js"; import { PostgresDatabaseService } from "@/database/postgres/service.js"; import type { DatabaseService } from "@/database/service.js"; @@ -10,14 +10,12 @@ import { createServer } from "@/server/service.js"; */ export async function runServer({ common, - indexingBuild, - apiBuild, + build, }: { common: Common; - indexingBuild: IndexingBuild; - apiBuild: ApiBuild | undefined; + build: ApiBuild; }) { - const { databaseConfig, optionsConfig, schema } = indexingBuild; + const { databaseConfig, optionsConfig, schema } = build; common.options = { ...common.options, ...optionsConfig }; @@ -37,8 +35,8 @@ export async function runServer({ } const server = await createServer({ - app: apiBuild?.app, - routes: apiBuild?.routes, + app: build.app, + routes: build.routes, common, schema, database, diff --git a/packages/core/src/build/index.ts b/packages/core/src/build/index.ts index 8f5e98c1a..5bbcc2c03 100644 --- a/packages/core/src/build/index.ts +++ b/packages/core/src/build/index.ts @@ -1,5 +1,5 @@ import { type Extend, extend } from "@/utils/extend.js"; -import { create, kill, start, startServer } from "./service.js"; +import { create, kill, start } from "./service.js"; import type { ApiBuild, ApiBuildResult, @@ -8,7 +8,7 @@ import type { Service, } from "./service.js"; -const methods = { start, startServer, kill }; +const methods = { start, kill }; export const createBuildService = extend(create, methods); diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index eb5628497..a3ce3de44 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -44,7 +44,7 @@ export type Service = { viteNodeRunner: ViteNodeRunner; }; -export type IndexingBuild = { +type BaseBuild = { // Build ID for caching buildId: string; // Config @@ -55,13 +55,15 @@ export type IndexingBuild = { // Schema schema: Schema; graphqlSchema: GraphQLSchema; - // Indexing functions +}; + +export type IndexingBuild = BaseBuild & { indexingFunctions: IndexingFunctions; }; -export type ApiBuild = { - app: Hono; - routes: PonderRoutes; +export type ApiBuild = BaseBuild & { + app?: Hono; + routes?: PonderRoutes; }; export type IndexingBuildResult = @@ -69,18 +71,9 @@ export type IndexingBuildResult = | { status: "error"; error: Error }; export type ApiBuildResult = - | { status: "success"; build?: ApiBuild } + | { status: "success"; build: ApiBuild } | { status: "error"; error: Error }; -type RawBuild = { - config: { config: Config; contentHash: string }; - schema: { schema: Schema; contentHash: string }; - indexingFunctions: { - indexingFunctions: RawIndexingFunctions; - contentHash: string; - }; -}; - export const create = async ({ common, }: { @@ -180,36 +173,54 @@ export const start = async ( buildService: Service, { watch, - onBuild, + onIndexingBuild, + onApiBuild, }: - | { watch: true; onBuild: (buildResult: IndexingBuildResult) => void } - | { watch: false; onBuild?: never }, -): Promise => { + | { + watch: true; + onIndexingBuild: (buildResult: IndexingBuildResult) => void; + onApiBuild: (buildResult: ApiBuildResult) => void; + } + | { watch: false; onIndexingBuild?: never; onApiBuild?: never }, +): Promise<{ indexing: IndexingBuildResult; api: ApiBuildResult }> => { const { common } = buildService; // Note: Don't run these in parallel. If there are circular imports in user code, // it's possible for ViteNodeRunner to return exports as undefined (a race condition). const configResult = await executeConfig(buildService); const schemaResult = await executeSchema(buildService); - const indexingFunctionsResult = await executeIndexingFunctions(buildService); + const indexingResult = await executeIndexingFunctions(buildService); + const apiResult = await executeApiRoutes(buildService); if (configResult.status === "error") { - return { status: "error", error: configResult.error }; + return { + indexing: { status: "error", error: configResult.error }, + api: { status: "error", error: configResult.error }, + }; } if (schemaResult.status === "error") { - return { status: "error", error: schemaResult.error }; + return { + indexing: { status: "error", error: schemaResult.error }, + api: { status: "error", error: schemaResult.error }, + }; } - if (indexingFunctionsResult.status === "error") { - return { status: "error", error: indexingFunctionsResult.error }; + if (indexingResult.status === "error") { + return { + indexing: { status: "error", error: indexingResult.error }, + api: { status: "error", error: indexingResult.error }, + }; + } + if (apiResult.status === "error") { + return { + indexing: { status: "error", error: apiResult.error }, + api: { status: "error", error: apiResult.error }, + }; } - const rawBuild: RawBuild = { - config: configResult, - schema: schemaResult, - indexingFunctions: indexingFunctionsResult, - }; - - const buildResult = await validateAndBuild(buildService, rawBuild); + let cachedConfigResult = configResult; + let cachedSchemaResult = schemaResult; + let cachedIndexingResult = indexingResult; + let cachedApiResult = apiResult; // If watch is false (`ponder start` or `ponder serve`), // don't register any event handlers on the watcher. @@ -258,15 +269,23 @@ export const start = async ( const hasSchemaUpdate = invalidated.includes( common.options.schemaFile.replace(/\\/g, "/"), ); - const hasSrcUpdate = invalidated.some( + const hasIndexingUpdate = invalidated.some( (file) => buildService.indexingRegex.test(file) && !buildService.apiRegex.test(file), ); + const hasApiUpdate = invalidated.some((file) => + buildService.apiRegex.test(file), + ); // This branch could trigger if you change a `note.txt` file within `src/`. // Note: We could probably do a better job filtering out files in `isFileIgnored`. - if (!hasConfigUpdate && !hasSchemaUpdate && !hasSrcUpdate) { + if ( + !hasConfigUpdate && + !hasSchemaUpdate && + !hasIndexingUpdate && + !hasApiUpdate + ) { return; } @@ -280,22 +299,22 @@ export const start = async ( if (hasConfigUpdate) { const result = await executeConfig(buildService); if (result.status === "error") { - onBuild({ status: "error", error: result.error }); + onIndexingBuild({ status: "error", error: result.error }); return; } - rawBuild.config = result; + cachedConfigResult = result; } if (hasSchemaUpdate) { const result = await executeSchema(buildService); if (result.status === "error") { - onBuild({ status: "error", error: result.error }); + onIndexingBuild({ status: "error", error: result.error }); return; } - rawBuild.schema = result; + cachedSchemaResult = result; } - if (hasSrcUpdate) { + if (hasIndexingUpdate) { const files = glob.sync(buildService.indexingPattern, { ignore: buildService.apiPattern, }); @@ -304,94 +323,108 @@ export const start = async ( const result = await executeIndexingFunctions(buildService); if (result.status === "error") { - onBuild({ status: "error", error: result.error }); + onIndexingBuild({ status: "error", error: result.error }); return; } - rawBuild.indexingFunctions = result; + cachedIndexingResult = result; } - const buildResult = await validateAndBuild(buildService, rawBuild); - onBuild(buildResult); - }; - - buildService.viteDevServer.watcher.on("change", onFileChange); - } - - return buildResult; -}; - -export const startServer = async ( - buildService: Service, - { - watch, - onBuild, - }: - | { watch: true; onBuild: (buildResult: ApiBuildResult) => void } - | { watch: false; onBuild?: never }, -): Promise => { - const { common } = buildService; - - const serverResult = await executeServer(buildService); - - if (serverResult.status === "error") { - return { status: "error", error: serverResult.error }; - } + if (hasApiUpdate) { + const files = glob.sync(buildService.apiPattern); + buildService.viteNodeRunner.moduleCache.invalidateDepTree(files); + buildService.viteNodeRunner.moduleCache.deleteByModuleId("@/generated"); - const buildResult = validateAndBuildServer(buildService, serverResult); + const result = await executeApiRoutes(buildService); + if (result.status === "error") { + onApiBuild({ status: "error", error: result.error }); + return; + } + cachedApiResult = result; + } - // If watch is false (`ponder start` or `ponder serve`), - // don't register any event handlers on the watcher. - if (watch) { - // Define the directories and files to ignore - const ignoredDirs = [common.options.generatedDir, common.options.ponderDir]; - const ignoredFiles = [ - path.join(common.options.rootDir, "ponder-env.d.ts"), - path.join(common.options.rootDir, ".env.local"), - ]; + /** + * Build and validate updated indexing and api artifacts + * + * There are a few cases to handle: + * 1) config or schema is updated -> rebuild both api and indexing + * 2) indexing functions are updated -> rebuild indexing + * 3) api routes are updated -> rebuild api + * + * Note: the api build cannot be successful if the indexing + * build fails, this means that any indexing errors are always + * propogated to the api build. + */ + + const indexingBuildResult = await validateAndBuild( + buildService, + cachedConfigResult, + cachedSchemaResult, + cachedIndexingResult, + ); + if (indexingBuildResult.status === "error") { + onIndexingBuild(indexingBuildResult); + onApiBuild(indexingBuildResult); + return; + } - const isFileIgnored = (filePath: string) => { - const isInIgnoredDir = ignoredDirs.some((dir) => { - const rel = path.relative(dir, filePath); - return !rel.startsWith("..") && !path.isAbsolute(rel); - }); + // If schema or config is updated, rebuild both api and indexing + if (hasConfigUpdate || hasSchemaUpdate) { + onIndexingBuild(indexingBuildResult); + onApiBuild( + validateAndBuildApi( + buildService, + indexingBuildResult.build, + cachedApiResult, + ), + ); + } else { + if (hasIndexingUpdate) { + onIndexingBuild(indexingBuildResult); + } - const isIgnoredFile = ignoredFiles.includes(filePath); - return isInIgnoredDir || isIgnoredFile; + if (hasApiUpdate) { + onApiBuild( + validateAndBuildApi( + buildService, + indexingBuildResult.build, + cachedApiResult, + ), + ); + } + } }; - const onFileChange = async (file: string) => { - if (isFileIgnored(file)) return; - - const files = glob.sync(buildService.apiPattern); - - if (files.includes(file) === false) return; - - // Invalidate all server modules. - buildService.viteNodeRunner.moduleCache.invalidateDepTree(files); - - common.logger.info({ - service: "build", - msg: `Hot reload ${files - .map((f) => `'${path.relative(common.options.rootDir, f)}'`) - .join(", ")}`, - }); - - buildService.viteNodeRunner.moduleCache.deleteByModuleId("@/generated"); + buildService.viteDevServer.watcher.on("change", onFileChange); + } - const result = await executeServer(buildService); + // Build and validate initial indexing and server build. + // Note: the api build cannot be successful if the indexing + // build fails - if (result.status === "error") { - onBuild({ status: "error", error: result.error }); - return; - } + const initialBuildResult = await validateAndBuild( + buildService, + configResult, + schemaResult, + indexingResult, + ); - onBuild(validateAndBuildServer(buildService, result)); + if (initialBuildResult.status === "error") { + return { + indexing: { status: "error", error: initialBuildResult.error }, + api: { status: "error", error: initialBuildResult.error }, }; - - buildService.viteDevServer.watcher.on("change", onFileChange); } - return buildResult; + const initialApiBuildResult = validateAndBuildApi( + buildService, + initialBuildResult.build, + apiResult, + ); + + return { + indexing: initialBuildResult, + api: initialApiBuildResult, + }; }; export const kill = async (buildService: Service): Promise => { @@ -521,7 +554,7 @@ const executeIndexingFunctions = async ( }; }; -const executeServer = async ( +const executeApiRoutes = async ( buildService: Service, ): Promise< | { @@ -574,11 +607,16 @@ const executeServer = async ( const validateAndBuild = async ( { common }: Pick, - rawBuild: RawBuild, + config: { config: Config; contentHash: string }, + schema: { schema: Schema; contentHash: string }, + indexingFunctions: { + indexingFunctions: RawIndexingFunctions; + contentHash: string; + }, ): Promise => { // Validate and build the schema const buildSchemaResult = safeBuildSchema({ - schema: rawBuild.schema.schema, + schema: schema.schema, }); if (buildSchemaResult.status === "error") { common.logger.error({ @@ -599,8 +637,8 @@ const validateAndBuild = async ( // Validates and build the config const buildConfigAndIndexingFunctionsResult = await safeBuildConfigAndIndexingFunctions({ - config: rawBuild.config.config, - rawIndexingFunctions: rawBuild.indexingFunctions.indexingFunctions, + config: config.config, + rawIndexingFunctions: indexingFunctions.indexingFunctions, options: common.options, }); if (buildConfigAndIndexingFunctionsResult.status === "error") { @@ -619,9 +657,9 @@ const validateAndBuild = async ( const buildId = createHash("sha256") .update(BUILD_ID_VERSION) - .update(rawBuild.config.contentHash) - .update(rawBuild.schema.contentHash) - .update(rawBuild.indexingFunctions.contentHash) + .update(config.contentHash) + .update(schema.contentHash) + .update(indexingFunctions.contentHash) .digest("hex") .slice(0, 10); @@ -646,17 +684,18 @@ const validateAndBuild = async ( }; }; -const validateAndBuildServer = ( +const validateAndBuildApi = ( { common }: Pick, - build: Partial, + baseBuild: BaseBuild, + api: { app?: Hono; routes?: PonderRoutes }, ): ApiBuildResult => { - if (!build.app || !build.routes) { - return { status: "success" }; + if (!api.app || !api.routes) { + return { status: "success", build: baseBuild }; } for (const { pathOrHandlers: [maybePathOrHandler], - } of build.routes) { + } of api.routes) { if (typeof maybePathOrHandler === "string") { if ( maybePathOrHandler === "/status" || @@ -668,7 +707,7 @@ const validateAndBuildServer = ( ); error.stack = undefined; common.logger.error({ service: "build", msg: "Failed build", error }); - return { status: "error", error }; + return { status: "error", error } as const; } } } @@ -676,8 +715,9 @@ const validateAndBuildServer = ( return { status: "success", build: { - app: build.app, - routes: build.routes, + ...baseBuild, + app: api.app, + routes: api.routes, }, }; }; From bb270da43d37b3ec4279a0f364cb1b10899675ad Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 15:06:04 -0400 Subject: [PATCH 104/122] :( --- package.json | 2 +- packages/core/package.json | 2 +- pnpm-lock.yaml | 52 +++++++++++++++++--------------------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/package.json b/package.json index 6eccf8d62..daa7d404a 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "@biomejs/biome": "^1.8.1", "@changesets/changelog-github": "^0.4.8", "@changesets/cli": "^2.26.2", - "hono": "4.0.0", + "hono": "4.5.0", "lint-staged": "^15.1.0", "simple-git-hooks": "^2.9.0", "typescript": "5.0.4", diff --git a/packages/core/package.json b/packages/core/package.json index 52a3689d7..7ee14e36b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -33,7 +33,7 @@ "typecheck": "tsc --noEmit" }, "peerDependencies": { - "hono": ">=4", + "hono": ">=4.5", "typescript": ">=5.0.4", "viem": ">=1.16" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b42199387..2b2dadc39 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,8 +23,8 @@ importers: specifier: ^2.26.2 version: 2.27.1 hono: - specifier: 4.0.0 - version: 4.0.0 + specifier: 4.5.0 + version: 4.5.0 lint-staged: specifier: ^15.1.0 version: 15.2.0 @@ -371,7 +371,7 @@ importers: version: link:../../packages/core hono: specifier: ^4.4.6 - version: 4.4.6 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3) @@ -443,7 +443,7 @@ importers: dependencies: forge-std: specifier: github:foundry-rs/forge-std - version: github.com/foundry-rs/forge-std/19891e6a0b5474b9ea6827ddb90bb9388f7acfc0 + version: github.com/foundry-rs/forge-std/07263d193d621c4b2b0ce8b4d54af58f6957d97d examples/with-foundry/ponder: dependencies: @@ -8450,7 +8450,7 @@ packages: devlop: 1.1.0 hast-util-from-parse5: 8.0.1 parse5: 7.1.2 - vfile: 6.0.1 + vfile: 6.0.2 vfile-message: 4.0.2 dev: false @@ -8462,7 +8462,7 @@ packages: devlop: 1.1.0 hastscript: 8.0.0 property-information: 6.5.0 - vfile: 6.0.1 + vfile: 6.0.2 vfile-location: 5.0.2 web-namespaces: 2.0.1 dev: false @@ -8479,8 +8479,8 @@ packages: '@types/hast': 3.0.4 dev: false - /hast-util-raw@9.0.3: - resolution: {integrity: sha512-ICWvVOF2fq4+7CMmtCPD5CM4QKjPbHpPotE6+8tDooV0ZuyJVUzHsrNX+O5NaRbieTf0F7FfeBOMAwi6Td0+yQ==} + /hast-util-raw@9.0.4: + resolution: {integrity: sha512-LHE65TD2YiNsHD3YuXcKPHXPLuYh/gjp12mOfU8jxSrm1f/yJpsb0F/KKljS6U9LJoP0Ux+tCe8iJ2AsPzTdgA==} dependencies: '@types/hast': 3.0.4 '@types/unist': 3.0.2 @@ -8492,7 +8492,7 @@ packages: parse5: 7.1.2 unist-util-position: 5.0.0 unist-util-visit: 5.0.0 - vfile: 6.0.1 + vfile: 6.0.2 web-namespaces: 2.0.1 zwitch: 2.0.4 dev: false @@ -8549,7 +8549,7 @@ packages: '@types/unist': 3.0.2 ccount: 2.0.1 comma-separated-tokens: 2.0.3 - hast-util-raw: 9.0.3 + hast-util-raw: 9.0.4 hast-util-whitespace: 3.0.0 html-void-elements: 3.0.0 mdast-util-to-hast: 13.2.0 @@ -8619,15 +8619,9 @@ packages: minimalistic-crypto-utils: 1.0.1 dev: true - /hono@4.0.0: - resolution: {integrity: sha512-8dKhuBBpRZEodUttQhrSFJ6PQqHRjXHyeeegfxOf132pvgbf0tOb9qqb7q7eYwAWpOcYrsUOsWdJ0sQIIovhZg==} + /hono@4.5.0: + resolution: {integrity: sha512-ZbezypZfn4odyApjCCv+Fw5OgweBqRLA/EsMyc4FUknFvBJcBIKhHy4sqmD1rWpBc/3wUlaQ6tqOPjk36R1ckg==} engines: {node: '>=16.0.0'} - dev: true - - /hono@4.4.6: - resolution: {integrity: sha512-XGRnoH8WONv60+PPvP9Sn067A9r/8JdHDJ5bgon0DVEHeR1cJPkWjv2aT+DBfMH9/mEkYa1+VEVFp1DT1lIwjw==} - engines: {node: '>=16.0.0'} - dev: false /hosted-git-info@2.8.9: resolution: {integrity: sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==} @@ -10203,7 +10197,7 @@ packages: trim-lines: 3.0.1 unist-util-position: 5.0.0 unist-util-visit: 5.0.0 - vfile: 6.0.1 + vfile: 6.0.2 dev: false /mdast-util-to-markdown@1.5.0: @@ -12502,7 +12496,7 @@ packages: hast-util-to-text: 4.0.0 katex: 0.16.9 unist-util-visit-parents: 6.0.1 - vfile: 6.0.1 + vfile: 6.0.2 dev: false /rehype-pretty-code@0.10.1(shikiji@0.6.10): @@ -12521,8 +12515,8 @@ packages: resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==} dependencies: '@types/hast': 3.0.4 - hast-util-raw: 9.0.3 - vfile: 6.0.1 + hast-util-raw: 9.0.4 + vfile: 6.0.2 dev: false /remark-frontmatter@4.0.1: @@ -14128,7 +14122,7 @@ packages: extend: 3.0.2 is-plain-obj: 4.1.0 trough: 2.1.0 - vfile: 6.0.1 + vfile: 6.0.2 dev: false /unist-util-find-after@5.0.0: @@ -14377,7 +14371,7 @@ packages: resolution: {integrity: sha512-NXPYyxyBSH7zB5U6+3uDdd6Nybz6o6/od9rk8bp9H8GR3L+cm/fC0uUTbqBmUTnMCUDslAGBOIKNfvvb+gGlDg==} dependencies: '@types/unist': 3.0.2 - vfile: 6.0.1 + vfile: 6.0.2 dev: false /vfile-message@3.1.4: @@ -14403,8 +14397,8 @@ packages: vfile-message: 3.1.4 dev: false - /vfile@6.0.1: - resolution: {integrity: sha512-1bYqc7pt6NIADBJ98UiG0Bn/CHIVOoZ/IyEkqIruLg0mE1BKzkOXY2D6CSqQIcKqgadppE5lrxgWXJmXd7zZJw==} + /vfile@6.0.2: + resolution: {integrity: sha512-zND7NlS8rJYb/sPqkb13ZvbbUoExdbi4w3SfRrMq6R3FvnLQmmfpajJNITuuYm6AZ5uao9vy4BAos3EXBPf2rg==} dependencies: '@types/unist': 3.0.2 unist-util-stringify-position: 4.0.0 @@ -15053,8 +15047,8 @@ packages: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} dev: false - github.com/foundry-rs/forge-std/19891e6a0b5474b9ea6827ddb90bb9388f7acfc0: - resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/19891e6a0b5474b9ea6827ddb90bb9388f7acfc0} + github.com/foundry-rs/forge-std/07263d193d621c4b2b0ce8b4d54af58f6957d97d: + resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/07263d193d621c4b2b0ce8b4d54af58f6957d97d} name: forge-std - version: 1.8.2 + version: 1.9.1 dev: false From 6e029c50aae8f829627b5fb004fc1cecfe3bbfa5 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 15:10:13 -0400 Subject: [PATCH 105/122] fix tests --- packages/core/src/server/service.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/core/src/server/service.test.ts b/packages/core/src/server/service.test.ts index cfd756627..f55786087 100644 --- a/packages/core/src/server/service.test.ts +++ b/packages/core/src/server/service.test.ts @@ -47,7 +47,7 @@ test("not healthy", async (context) => { dbNamespace: namespaceInfo.userNamespace, }); - const response = await server.hono.request("/_ponder/health"); + const response = await server.hono.request("/health"); expect(response.status).toBe(503); @@ -72,7 +72,7 @@ test("healthy", async (context) => { db: database.indexingDb, }).setStatus({}); - const response = await server.hono.request("/_ponder/health"); + const response = await server.hono.request("/health"); expect(response.status).toBe(200); @@ -94,7 +94,7 @@ test("healthy PUT", async (context) => { dbNamespace: namespaceInfo.userNamespace, }); - const response = await server.hono.request("/_ponder/health", { + const response = await server.hono.request("/health", { method: "PUT", }); @@ -115,7 +115,7 @@ test("metrics", async (context) => { dbNamespace: namespaceInfo.userNamespace, }); - const response = await server.hono.request("/_ponder/metrics"); + const response = await server.hono.request("/metrics"); expect(response.status).toBe(200); @@ -137,7 +137,7 @@ test("metrics error", async (context) => { const metricsSpy = vi.spyOn(context.common.metrics, "getMetrics"); metricsSpy.mockRejectedValueOnce(new Error()); - const response = await server.hono.request("/_ponder/metrics"); + const response = await server.hono.request("/metrics"); expect(response.status).toBe(500); @@ -156,7 +156,7 @@ test("metrics PUT", async (context) => { dbNamespace: namespaceInfo.userNamespace, }); - const response = await server.hono.request("/_ponder/metrics", { + const response = await server.hono.request("/metrics", { method: "PUT", }); @@ -199,5 +199,5 @@ test.skip("kill", async (context) => { await server.kill(); - expect(() => server.hono.request("/_ponder/health")).rejects.toThrow(); + expect(() => server.hono.request("/health")).rejects.toThrow(); }); From 17a27d640b5caabcbb6f989cd40a47e0f37c3481 Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:34:15 -0400 Subject: [PATCH 106/122] fix graphql error status code --- packages/core/src/graphql/index.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/core/src/graphql/index.ts b/packages/core/src/graphql/index.ts index 48e5bcfbb..1c0a296b7 100644 --- a/packages/core/src/graphql/index.ts +++ b/packages/core/src/graphql/index.ts @@ -2,7 +2,7 @@ import { graphiQLHtml } from "@/ui/graphiql.html.js"; import { maxAliasesPlugin } from "@escape.tech/graphql-armor-max-aliases"; import { maxDepthPlugin } from "@escape.tech/graphql-armor-max-depth"; import { maxTokensPlugin } from "@escape.tech/graphql-armor-max-tokens"; -import { createYoga } from "graphql-yoga"; +import { type YogaServerInstance, createYoga } from "graphql-yoga"; import { createMiddleware } from "hono/factory"; import { buildGraphQLSchema } from "./buildGraphqlSchema.js"; import { buildLoaderCache } from "./buildLoaderCache.js"; @@ -36,7 +36,7 @@ export const graphql = ( maxOperationAliases: 30, }, ) => { - let yoga: any | undefined = undefined; + let yoga: YogaServerInstance | undefined = undefined; return createMiddleware(async (c) => { if (c.req.method === "GET") { @@ -74,6 +74,13 @@ export const graphql = ( }); } - return yoga.handle(c.req.raw); + const response = await yoga.handle(c.req.raw); + // TODO: Figure out why Yoga is returning 500 status codes for GraphQL errors. + // @ts-expect-error + response.status = 200; + // @ts-expect-error + response.statusText = "OK"; + + return response; }); }; From b5b38690095db593b2956b57b6261896377405da Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 20:39:24 -0400 Subject: [PATCH 107/122] trpc example and add drizzle objects to hono context --- examples/with-trpc/client/index.ts | 15 + examples/with-trpc/client/package.json | 11 + examples/with-trpc/client/tsconfig.json | 26 ++ examples/with-trpc/package.json | 8 + examples/with-trpc/ponder/.env.example | 5 + examples/with-trpc/ponder/.eslintrc.json | 3 + examples/with-trpc/ponder/.gitignore | 18 + examples/with-trpc/ponder/abis/erc20ABI.ts | 147 +++++++++ examples/with-trpc/ponder/package.json | 28 ++ examples/with-trpc/ponder/ponder-env.d.ts | 28 ++ examples/with-trpc/ponder/ponder.config.ts | 21 ++ examples/with-trpc/ponder/ponder.schema.ts | 50 +++ examples/with-trpc/ponder/src/api/index.ts | 32 ++ examples/with-trpc/ponder/src/index.ts | 70 ++++ examples/with-trpc/ponder/tsconfig.json | 26 ++ packages/core/src/hono/context.ts | 28 +- packages/core/src/hono/handler.ts | 90 ++--- packages/core/src/hono/index.ts | 9 +- packages/core/src/server/service.ts | 10 +- packages/core/src/types/api.ts | 27 ++ packages/core/src/types/hono.ts | 13 - packages/core/src/types/virtual.test-d.ts | 7 + packages/core/src/types/virtual.ts | 6 +- pnpm-lock.yaml | 366 +++++++++++++++++++-- 24 files changed, 933 insertions(+), 111 deletions(-) create mode 100644 examples/with-trpc/client/index.ts create mode 100644 examples/with-trpc/client/package.json create mode 100644 examples/with-trpc/client/tsconfig.json create mode 100644 examples/with-trpc/package.json create mode 100644 examples/with-trpc/ponder/.env.example create mode 100644 examples/with-trpc/ponder/.eslintrc.json create mode 100644 examples/with-trpc/ponder/.gitignore create mode 100644 examples/with-trpc/ponder/abis/erc20ABI.ts create mode 100644 examples/with-trpc/ponder/package.json create mode 100644 examples/with-trpc/ponder/ponder-env.d.ts create mode 100644 examples/with-trpc/ponder/ponder.config.ts create mode 100644 examples/with-trpc/ponder/ponder.schema.ts create mode 100644 examples/with-trpc/ponder/src/api/index.ts create mode 100644 examples/with-trpc/ponder/src/index.ts create mode 100644 examples/with-trpc/ponder/tsconfig.json create mode 100644 packages/core/src/types/api.ts delete mode 100644 packages/core/src/types/hono.ts diff --git a/examples/with-trpc/client/index.ts b/examples/with-trpc/client/index.ts new file mode 100644 index 000000000..3fa7d7d6e --- /dev/null +++ b/examples/with-trpc/client/index.ts @@ -0,0 +1,15 @@ +import { createTRPCProxyClient, httpBatchLink } from "@trpc/client"; +import { zeroAddress } from "viem"; +import type { AppRouter } from "../ponder/src/api/index"; + +const client = createTRPCProxyClient({ + links: [ + httpBatchLink({ + url: "http://localhost:42069/trpc", + }), + ], +}); + +const response = await client.hello.query(zeroAddress); + +console.log(response); diff --git a/examples/with-trpc/client/package.json b/examples/with-trpc/client/package.json new file mode 100644 index 000000000..881e96934 --- /dev/null +++ b/examples/with-trpc/client/package.json @@ -0,0 +1,11 @@ +{ + "private": true, + "name": "ponder-examples-with-trpc-ponder", + "type": "module", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "dependencies": { + "@trpc/client": "^10.45.2" + } +} diff --git a/examples/with-trpc/client/tsconfig.json b/examples/with-trpc/client/tsconfig.json new file mode 100644 index 000000000..592b9a939 --- /dev/null +++ b/examples/with-trpc/client/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + // Type checking + "strict": true, + "noUncheckedIndexedAccess": true, + + // Interop constraints + "verbatimModuleSyntax": false, + "esModuleInterop": true, + "isolatedModules": true, + "allowSyntheticDefaultImports": true, + "resolveJsonModule": true, + + // Language and environment + "moduleResolution": "bundler", + "module": "ESNext", + "noEmit": true, + "lib": ["ES2022"], + "target": "ES2022", + + // Skip type checking for node modules + "skipLibCheck": true + }, + "include": ["./**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/examples/with-trpc/package.json b/examples/with-trpc/package.json new file mode 100644 index 000000000..4873fe900 --- /dev/null +++ b/examples/with-trpc/package.json @@ -0,0 +1,8 @@ +{ + "name": "ponder-examples-with-trpc", + "private": true, + "type": "module", + "engines": { + "node": ">=18.14" + } +} diff --git a/examples/with-trpc/ponder/.env.example b/examples/with-trpc/ponder/.env.example new file mode 100644 index 000000000..f7745c21c --- /dev/null +++ b/examples/with-trpc/ponder/.env.example @@ -0,0 +1,5 @@ +# Mainnet RPC URL used for fetching blockchain data. Alchemy is recommended. +PONDER_RPC_URL_1=https://eth-mainnet.g.alchemy.com/v2/... + +# (Optional) Postgres database URL. If not provided, SQLite will be used. +DATABASE_URL= \ No newline at end of file diff --git a/examples/with-trpc/ponder/.eslintrc.json b/examples/with-trpc/ponder/.eslintrc.json new file mode 100644 index 000000000..359e2bbfa --- /dev/null +++ b/examples/with-trpc/ponder/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "ponder" +} diff --git a/examples/with-trpc/ponder/.gitignore b/examples/with-trpc/ponder/.gitignore new file mode 100644 index 000000000..f0c7e1177 --- /dev/null +++ b/examples/with-trpc/ponder/.gitignore @@ -0,0 +1,18 @@ +# Dependencies +/node_modules + +# Debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# Misc +.DS_Store + +# Env files +.env*.local + +# Ponder +/generated/ +/.ponder/ diff --git a/examples/with-trpc/ponder/abis/erc20ABI.ts b/examples/with-trpc/ponder/abis/erc20ABI.ts new file mode 100644 index 000000000..94cbc6a33 --- /dev/null +++ b/examples/with-trpc/ponder/abis/erc20ABI.ts @@ -0,0 +1,147 @@ +export const erc20ABI = [ + { + stateMutability: "view", + type: "function", + inputs: [], + name: "DOMAIN_SEPARATOR", + outputs: [{ name: "", internalType: "bytes32", type: "bytes32" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [ + { name: "", internalType: "address", type: "address" }, + { name: "", internalType: "address", type: "address" }, + ], + name: "allowance", + outputs: [{ name: "", internalType: "uint256", type: "uint256" }], + }, + { + stateMutability: "nonpayable", + type: "function", + inputs: [ + { name: "spender", internalType: "address", type: "address" }, + { name: "amount", internalType: "uint256", type: "uint256" }, + ], + name: "approve", + outputs: [{ name: "", internalType: "bool", type: "bool" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [{ name: "", internalType: "address", type: "address" }], + name: "balanceOf", + outputs: [{ name: "", internalType: "uint256", type: "uint256" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [], + name: "decimals", + outputs: [{ name: "", internalType: "uint8", type: "uint8" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [], + name: "name", + outputs: [{ name: "", internalType: "string", type: "string" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [{ name: "", internalType: "address", type: "address" }], + name: "nonces", + outputs: [{ name: "", internalType: "uint256", type: "uint256" }], + }, + { + stateMutability: "nonpayable", + type: "function", + inputs: [ + { name: "owner", internalType: "address", type: "address" }, + { name: "spender", internalType: "address", type: "address" }, + { name: "value", internalType: "uint256", type: "uint256" }, + { name: "deadline", internalType: "uint256", type: "uint256" }, + { name: "v", internalType: "uint8", type: "uint8" }, + { name: "r", internalType: "bytes32", type: "bytes32" }, + { name: "s", internalType: "bytes32", type: "bytes32" }, + ], + name: "permit", + outputs: [], + }, + { + stateMutability: "view", + type: "function", + inputs: [], + name: "symbol", + outputs: [{ name: "", internalType: "string", type: "string" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [], + name: "totalSupply", + outputs: [{ name: "", internalType: "uint256", type: "uint256" }], + }, + { + stateMutability: "nonpayable", + type: "function", + inputs: [ + { name: "to", internalType: "address", type: "address" }, + { name: "amount", internalType: "uint256", type: "uint256" }, + ], + name: "transfer", + outputs: [{ name: "", internalType: "bool", type: "bool" }], + }, + { + stateMutability: "nonpayable", + type: "function", + inputs: [ + { name: "from", internalType: "address", type: "address" }, + { name: "to", internalType: "address", type: "address" }, + { name: "amount", internalType: "uint256", type: "uint256" }, + ], + name: "transferFrom", + outputs: [{ name: "", internalType: "bool", type: "bool" }], + }, + { + type: "event", + anonymous: false, + inputs: [ + { + name: "owner", + internalType: "address", + type: "address", + indexed: true, + }, + { + name: "spender", + internalType: "address", + type: "address", + indexed: true, + }, + { + name: "amount", + internalType: "uint256", + type: "uint256", + indexed: false, + }, + ], + name: "Approval", + }, + { + type: "event", + anonymous: false, + inputs: [ + { name: "from", internalType: "address", type: "address", indexed: true }, + { name: "to", internalType: "address", type: "address", indexed: true }, + { + name: "amount", + internalType: "uint256", + type: "uint256", + indexed: false, + }, + ], + name: "Transfer", + }, +] as const; diff --git a/examples/with-trpc/ponder/package.json b/examples/with-trpc/ponder/package.json new file mode 100644 index 000000000..7c9c27c34 --- /dev/null +++ b/examples/with-trpc/ponder/package.json @@ -0,0 +1,28 @@ +{ + "private": true, + "name": "ponder-examples-with-trpc-ponder", + "type": "module", + "scripts": { + "dev": "ponder dev", + "start": "ponder start", + "codegen": "ponder codegen", + "lint": "eslint .", + "typecheck": "tsc" + }, + "dependencies": { + "@hono/trpc-server": "^0.3.2", + "@ponder/core": "workspace:*", + "@trpc/server": "^10.45.2", + "viem": "^1.19.9", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/node": "^20.10.0", + "eslint": "^8.54.0", + "eslint-config-ponder": "workspace:*", + "typescript": "^5.3.2" + }, + "engines": { + "node": ">=18.14" + } +} diff --git a/examples/with-trpc/ponder/ponder-env.d.ts b/examples/with-trpc/ponder/ponder-env.d.ts new file mode 100644 index 000000000..a99dcf263 --- /dev/null +++ b/examples/with-trpc/ponder/ponder-env.d.ts @@ -0,0 +1,28 @@ +// This file enables type checking and editor autocomplete for this Ponder project. +// After upgrading, you may find that changes have been made to this file. +// If this happens, please commit the changes. Do not manually edit this file. +// See https://ponder.sh/docs/getting-started/installation#typescript for more information. + +declare module "@/generated" { + import type { Virtual } from "@ponder/core"; + + type config = typeof import("./ponder.config.ts").default; + type schema = typeof import("./ponder.schema.ts").default; + + export const ponder: Virtual.Registry; + + export type EventNames = Virtual.EventNames; + export type Event = Virtual.Event< + config, + name + >; + export type Context = Virtual.Context< + config, + schema, + name + >; + export type IndexingFunctionArgs = + Virtual.IndexingFunctionArgs; + export type ApiContext = Virtual.Drizzle; + export type Schema = Virtual.Schema; +} diff --git a/examples/with-trpc/ponder/ponder.config.ts b/examples/with-trpc/ponder/ponder.config.ts new file mode 100644 index 000000000..32ae5a2ba --- /dev/null +++ b/examples/with-trpc/ponder/ponder.config.ts @@ -0,0 +1,21 @@ +import { createConfig } from "@ponder/core"; +import { http } from "viem"; +import { erc20ABI } from "./abis/erc20ABI"; + +export default createConfig({ + networks: { + mainnet: { + chainId: 1, + transport: http(process.env.PONDER_RPC_URL_1), + }, + }, + contracts: { + ERC20: { + network: "mainnet", + abi: erc20ABI, + address: "0x32353A6C91143bfd6C7d363B546e62a9A2489A20", + startBlock: 13142655, + endBlock: 13150000, + }, + }, +}); diff --git a/examples/with-trpc/ponder/ponder.schema.ts b/examples/with-trpc/ponder/ponder.schema.ts new file mode 100644 index 000000000..a722500c6 --- /dev/null +++ b/examples/with-trpc/ponder/ponder.schema.ts @@ -0,0 +1,50 @@ +import { createSchema } from "@ponder/core"; + +export default createSchema((p) => ({ + Account: p.createTable({ + id: p.hex(), + balance: p.bigint(), + isOwner: p.boolean(), + + allowances: p.many("Allowance.ownerId"), + approvalOwnerEvents: p.many("ApprovalEvent.ownerId"), + approvalSpenderEvents: p.many("ApprovalEvent.spenderId"), + transferFromEvents: p.many("TransferEvent.fromId"), + transferToEvents: p.many("TransferEvent.toId"), + }), + Allowance: p.createTable({ + id: p.string(), + amount: p.bigint(), + + ownerId: p.hex().references("Account.id"), + spenderId: p.hex().references("Account.id"), + + owner: p.one("ownerId"), + spender: p.one("spenderId"), + }), + TransferEvent: p.createTable( + { + id: p.string(), + amount: p.bigint(), + timestamp: p.int(), + + fromId: p.hex().references("Account.id"), + toId: p.hex().references("Account.id"), + + from: p.one("fromId"), + to: p.one("toId"), + }, + { fromIdIndex: p.index("fromId") }, + ), + ApprovalEvent: p.createTable({ + id: p.string(), + amount: p.bigint(), + timestamp: p.int(), + + ownerId: p.hex().references("Account.id"), + spenderId: p.hex().references("Account.id"), + + owner: p.one("ownerId"), + spender: p.one("spenderId"), + }), +})); diff --git a/examples/with-trpc/ponder/src/api/index.ts b/examples/with-trpc/ponder/src/api/index.ts new file mode 100644 index 000000000..d8babe0e9 --- /dev/null +++ b/examples/with-trpc/ponder/src/api/index.ts @@ -0,0 +1,32 @@ +import { type ApiContext, ponder } from "@/generated"; +import { trpcServer } from "@hono/trpc-server"; +import { eq } from "@ponder/core"; +import { initTRPC } from "@trpc/server"; +import type { Address } from "viem"; +import { z } from "zod"; + +const t = initTRPC.context().create(); + +const appRouter = t.router({ + hello: t.procedure.input(z.string()).query(async ({ input, ctx }) => { + const { Account } = ctx.tables; + + const account = await ctx.db + .select({ balance: Account.balance }) + .from(Account) + .where(eq(Account.id, input as Address)) + .limit(1); + + return account[0]?.balance?.toString() ?? null; + }), +}); + +export type AppRouter = typeof appRouter; + +ponder.use( + "/trpc/*", + trpcServer({ + router: appRouter, + createContext: (_, c) => ({ db: c.env.db, tables: c.env.tables }), + }), +); diff --git a/examples/with-trpc/ponder/src/index.ts b/examples/with-trpc/ponder/src/index.ts new file mode 100644 index 000000000..16bf33aa7 --- /dev/null +++ b/examples/with-trpc/ponder/src/index.ts @@ -0,0 +1,70 @@ +import { ponder } from "@/generated"; + +ponder.on("ERC20:Transfer", async ({ event, context }) => { + const { Account, TransferEvent } = context.db; + + // Create an Account for the sender, or update the balance if it already exists. + await Account.upsert({ + id: event.args.from, + create: { + balance: BigInt(0), + isOwner: false, + }, + update: ({ current }) => ({ + balance: current.balance - event.args.amount, + }), + }); + + // Create an Account for the recipient, or update the balance if it already exists. + await Account.upsert({ + id: event.args.to, + create: { + balance: event.args.amount, + isOwner: false, + }, + update: ({ current }) => ({ + balance: current.balance + event.args.amount, + }), + }); + + // Create a TransferEvent. + await TransferEvent.create({ + id: event.log.id, + data: { + fromId: event.args.from, + toId: event.args.to, + amount: event.args.amount, + timestamp: Number(event.block.timestamp), + }, + }); +}); + +ponder.on("ERC20:Approval", async ({ event, context }) => { + const { Allowance, ApprovalEvent } = context.db; + + const allowanceId = `${event.args.owner}-${event.args.spender}`; + + // Create or update the Allowance. + await Allowance.upsert({ + id: allowanceId, + create: { + ownerId: event.args.owner, + spenderId: event.args.spender, + amount: event.args.amount, + }, + update: { + amount: event.args.amount, + }, + }); + + // Create an ApprovalEvent. + await ApprovalEvent.create({ + id: event.log.id, + data: { + ownerId: event.args.owner, + spenderId: event.args.spender, + amount: event.args.amount, + timestamp: Number(event.block.timestamp), + }, + }); +}); diff --git a/examples/with-trpc/ponder/tsconfig.json b/examples/with-trpc/ponder/tsconfig.json new file mode 100644 index 000000000..592b9a939 --- /dev/null +++ b/examples/with-trpc/ponder/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + // Type checking + "strict": true, + "noUncheckedIndexedAccess": true, + + // Interop constraints + "verbatimModuleSyntax": false, + "esModuleInterop": true, + "isolatedModules": true, + "allowSyntheticDefaultImports": true, + "resolveJsonModule": true, + + // Language and environment + "moduleResolution": "bundler", + "module": "ESNext", + "noEmit": true, + "lib": ["ES2022"], + "target": "ES2022", + + // Skip type checking for node modules + "skipLibCheck": true + }, + "include": ["./**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/packages/core/src/hono/context.ts b/packages/core/src/hono/context.ts index 8fe3a0a3d..067b2dfc7 100644 --- a/packages/core/src/hono/context.ts +++ b/packages/core/src/hono/context.ts @@ -1,22 +1,12 @@ -import type { DrizzleDb } from "@/drizzle/db.js"; -import type { DrizzleTable } from "@/drizzle/table.js"; -import type { ExtractTableNames, Schema } from "@/schema/common.js"; +import type { Schema } from "@/schema/common.js"; +import type { ApiContext } from "@/types/api.js"; import type { Env, Context as HonoContext, Input } from "hono"; export type Context< schema extends Schema = Schema, path extends string = string, input extends Input = {}, -> = { - db: DrizzleDb; - tables: { - [tableName in ExtractTableNames]: DrizzleTable< - tableName, - // @ts-ignore - schema[tableName]["table"], - schema - >; - }; +> = ApiContext & { /** * Hono request object. * @@ -59,14 +49,4 @@ export type MiddlewareContext< schema extends Schema = Schema, path extends string = string, input extends Input = {}, -> = HonoContext & { - db: DrizzleDb; - tables: { - [tableName in ExtractTableNames]: DrizzleTable< - tableName, - // @ts-ignore - schema[tableName]["table"], - schema - >; - }; -}; +> = ApiContext & HonoContext; diff --git a/packages/core/src/hono/handler.ts b/packages/core/src/hono/handler.ts index 5bb389b3e..725a95764 100644 --- a/packages/core/src/hono/handler.ts +++ b/packages/core/src/hono/handler.ts @@ -1,5 +1,5 @@ import type { Schema } from "@/schema/common.js"; -import type { PonderHono } from "@/types/hono.js"; +import type { ApiRegistry } from "@/types/api.js"; import type { BlankInput, HandlerResponse, Input, Next } from "hono/types"; import type { Context, MiddlewareContext } from "./context.js"; @@ -29,7 +29,7 @@ export type HandlerInterface = { response extends HandlerResponse = any, >( handler: Handler, - ): PonderHono; + ): ApiRegistry; // app.get(handler x2) < @@ -42,7 +42,7 @@ export type HandlerInterface = { Handler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler) < @@ -52,7 +52,7 @@ export type HandlerInterface = { >( path: path, handler: Handler, - ): PonderHono; + ): ApiRegistry; // app.get(handler x 3) < @@ -67,7 +67,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x2) < @@ -81,7 +81,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(handler x 4) < @@ -98,7 +98,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x3) < @@ -114,7 +114,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(handler x 5) < @@ -133,7 +133,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x4) < @@ -151,7 +151,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(handler x 6) < @@ -172,7 +172,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x5) < @@ -192,7 +192,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(handler x 7) < @@ -215,7 +215,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x6) < @@ -237,7 +237,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(handler x 8) < @@ -268,7 +268,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x7) < @@ -292,7 +292,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(handler x 9) < @@ -332,7 +332,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x8) < @@ -364,7 +364,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(handler x 10) < @@ -414,7 +414,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x9) < @@ -455,7 +455,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x10) < @@ -506,7 +506,7 @@ export type HandlerInterface = { MiddlewareHandler, Handler, ] - ): PonderHono; + ): ApiRegistry; // app.get(...handlers[]) < @@ -515,7 +515,7 @@ export type HandlerInterface = { response extends HandlerResponse = any, >( ...handlers: Handler[] - ): PonderHono; + ): ApiRegistry; // app.get(path, ...handlers[]) < @@ -525,18 +525,18 @@ export type HandlerInterface = { >( path: path, ...handlers: Handler[] - ): PonderHono; + ): ApiRegistry; // app.get(path) - (path: path): PonderHono; + (path: path): ApiRegistry; }; export interface MiddlewareHandlerInterface { //// app.use(...handlers[]) - (...handlers: MiddlewareHandler[]): PonderHono; + (...handlers: MiddlewareHandler[]): ApiRegistry; // app.use(handler) - (handler: MiddlewareHandler): PonderHono; + (handler: MiddlewareHandler): ApiRegistry; // app.use(handler x2) ( @@ -544,13 +544,13 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler) ( path: path, handler: MiddlewareHandler, - ): PonderHono; + ): ApiRegistry; // app.use(handler x3) ( @@ -559,7 +559,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x2) ( @@ -568,7 +568,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.use(handler x4) ( @@ -578,7 +578,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x3) ( @@ -588,7 +588,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.use(handler x5) ( @@ -599,7 +599,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x4) ( @@ -610,7 +610,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.use(handler x6) ( @@ -622,7 +622,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x5) ( @@ -634,7 +634,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.use(handler x7) ( @@ -647,7 +647,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x6) ( @@ -660,7 +660,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.use(handler x8) ( @@ -674,7 +674,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x7) ( @@ -688,7 +688,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.use(handler x9) ( @@ -703,7 +703,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x8) ( @@ -718,7 +718,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.use(handler x10) ( @@ -734,7 +734,7 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; // app.get(path, handler x9) ( @@ -750,11 +750,11 @@ export interface MiddlewareHandlerInterface { MiddlewareHandler, MiddlewareHandler, ] - ): PonderHono; + ): ApiRegistry; //// app.use(path, ...handlers[]) ( path: path, ...handlers: MiddlewareHandler[] - ): PonderHono; + ): ApiRegistry; } diff --git a/packages/core/src/hono/index.ts b/packages/core/src/hono/index.ts index 83f0a4a40..6674fa16c 100644 --- a/packages/core/src/hono/index.ts +++ b/packages/core/src/hono/index.ts @@ -17,7 +17,12 @@ export const applyHonoRoutes = ( // add custom properties to hono context const addCustomContext = (handler: Handler | MiddlewareHandler) => (c: any, next: any) => { - return handler({ ...customContext, ...c }, next); + for (const key of Object.keys(customContext ?? {})) { + // @ts-ignore + c[key] = customContext![key]; + } + + return handler(c, next); }; for (const { @@ -52,7 +57,7 @@ export const applyHonoRoutes = ( } for (const handler of handlers) { // @ts-expect-error access private property - hono.addRoute("ALL", path, handler); + hono.addRoute("ALL", path, addCustomContext(handler)); } } } diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 885085fe8..9a9af1dcb 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -90,8 +90,13 @@ export async function createServer({ } }); - // context required for graphql middleware + const db = createDrizzleDb(database); + const tables = createDrizzleTables(schema, database, dbNamespace); + + // context required for graphql middleware and hono middleware const contextMiddleware = createMiddleware(async (c, next) => { + c.set("db", db); + c.set("tables", tables); c.set("readonlyStore", readonlyStore); c.set("metadataStore", metadataStore); c.set("schema", schema); @@ -140,9 +145,6 @@ export async function createServer({ .use(contextMiddleware); if (userApp !== undefined && userRoutes !== undefined) { - const db = createDrizzleDb(database); - const tables = createDrizzleTables(schema, database, dbNamespace); - // apply user routes to hono instance, registering a custom error handler applyHonoRoutes(hono, userRoutes, { db, tables }).onError((error, c) => onError(error, c, common), diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.ts new file mode 100644 index 000000000..49b77ce4d --- /dev/null +++ b/packages/core/src/types/api.ts @@ -0,0 +1,27 @@ +import type { DrizzleDb } from "@/drizzle/db.js"; +import type { DrizzleTable } from "@/drizzle/table.js"; +import type { + HandlerInterface, + MiddlewareHandlerInterface, +} from "@/hono/handler.js"; +import type { ExtractTableNames, Schema } from "@/schema/common.js"; +import type { Hono } from "hono"; + +export type ApiContext = { + db: DrizzleDb; + tables: { + [tableName in ExtractTableNames]: DrizzleTable< + tableName, + // @ts-ignore + schema[tableName]["table"], + schema + >; + }; +}; + +export type ApiRegistry = { + get: HandlerInterface; + post: HandlerInterface; + use: MiddlewareHandlerInterface; + hono: Hono<{ Variables: ApiContext }>; +}; diff --git a/packages/core/src/types/hono.ts b/packages/core/src/types/hono.ts deleted file mode 100644 index 35ab9f6be..000000000 --- a/packages/core/src/types/hono.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { - HandlerInterface, - MiddlewareHandlerInterface, -} from "@/hono/handler.js"; -import type { Schema } from "@/schema/common.js"; -import type { Hono } from "hono"; - -export type PonderHono = { - get: HandlerInterface; - post: HandlerInterface; - use: MiddlewareHandlerInterface; - hono: Hono; -}; diff --git a/packages/core/src/types/virtual.test-d.ts b/packages/core/src/types/virtual.test-d.ts index 27a4dbee1..21eabf114 100644 --- a/packages/core/src/types/virtual.test-d.ts +++ b/packages/core/src/types/virtual.test-d.ts @@ -1,4 +1,5 @@ import { createConfig } from "@/config/config.js"; +import type { DrizzleDb } from "@/drizzle/db.js"; import { createSchema, createTable } from "@/schema/schema.js"; import { http, type Abi, type Address, type Hex, parseAbiItem } from "viem"; import { assertType, test } from "vitest"; @@ -437,3 +438,9 @@ test("Registry", () => { context.contracts.c2; }); }); + +test("Drizzle", () => { + type a = Virtual.Drizzle; + + assertType({} as any as { db: DrizzleDb; tables: { table: any } }); +}); diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index 01de4905c..6bea330eb 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -17,7 +17,7 @@ import type { TransactionReceipt, } from "@/types/eth.js"; import type { DatabaseModel } from "@/types/model.js"; -import type { PonderHono } from "./hono.js"; +import type { ApiRegistry, ApiContext as _ApiContext } from "./api.js"; import type { Prettify } from "./utils.js"; export namespace Virtual { @@ -227,6 +227,8 @@ export namespace Virtual { }; }; + export type Drizzle = _ApiContext; + export type IndexingFunctionArgs< config extends Config, schema extends BuilderSchema, @@ -247,5 +249,5 @@ export namespace Virtual { }, ) => Promise | void, ) => void; - } & PonderHono; + } & ApiRegistry; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2b2dadc39..5c8a527d0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -148,7 +148,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -170,7 +170,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -195,7 +195,7 @@ importers: version: 0.10.3(typescript@5.3.3) viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -217,7 +217,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -239,7 +239,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -261,7 +261,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -283,7 +283,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -305,7 +305,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -327,7 +327,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -349,7 +349,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -374,7 +374,7 @@ importers: version: 4.5.0 viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -396,7 +396,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -418,7 +418,7 @@ importers: version: link:../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.10.0 @@ -452,7 +452,7 @@ importers: version: link:../../../packages/core viem: specifier: ^1.19.3 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20.9.0 @@ -494,7 +494,7 @@ importers: version: 18.2.0(react@18.2.0) viem: specifier: ^1.19.11 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) devDependencies: '@types/node': specifier: ^20 @@ -531,7 +531,52 @@ importers: version: link:../../../packages/core viem: specifier: ^1.19.9 - version: 1.21.4(typescript@5.3.3) + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) + devDependencies: + '@types/node': + specifier: ^20.10.0 + version: 20.11.24 + eslint: + specifier: ^8.54.0 + version: 8.56.0 + eslint-config-ponder: + specifier: workspace:* + version: link:../../../packages/eslint-config-ponder + typescript: + specifier: ^5.3.2 + version: 5.3.3 + + examples/with-trpc: {} + + examples/with-trpc/client: + dependencies: + '@trpc/client': + specifier: ^10.45.2 + version: 10.45.2 + hono: + specifier: ^4.4.6 + version: 4.5.0 + tsx: + specifier: ^4.16.2 + version: 4.16.2 + + examples/with-trpc/ponder: + dependencies: + '@hono/trpc-server': + specifier: ^0.3.2 + version: 0.3.2(@trpc/server@10.45.2)(hono@4.5.0) + '@ponder/core': + specifier: workspace:* + version: link:../../../packages/core + '@trpc/server': + specifier: ^10.45.2 + version: 10.45.2 + viem: + specifier: ^1.19.9 + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) + zod: + specifier: ^3.23.8 + version: 3.23.8 devDependencies: '@types/node': specifier: ^20.10.0 @@ -2441,6 +2486,15 @@ packages: requiresBuild: true optional: true + /@esbuild/aix-ppc64@0.21.5: + resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [aix] + requiresBuild: true + dev: false + optional: true + /@esbuild/android-arm64@0.16.17: resolution: {integrity: sha512-MIGl6p5sc3RDTLLkYL1MyL8BMRN4tLMRCn+yRJJmEDvYZ2M7tmAf80hx1kbNEUX2KJ50RRtxZ4JHLvCfuB6kBg==} engines: {node: '>=12'} @@ -2458,6 +2512,15 @@ packages: requiresBuild: true optional: true + /@esbuild/android-arm64@0.21.5: + resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: false + optional: true + /@esbuild/android-arm@0.16.17: resolution: {integrity: sha512-N9x1CMXVhtWEAMS7pNNONyA14f71VPQN9Cnavj1XQh6T7bskqiLLrSca4O0Vr8Wdcga943eThxnVp3JLnBMYtw==} engines: {node: '>=12'} @@ -2475,6 +2538,15 @@ packages: requiresBuild: true optional: true + /@esbuild/android-arm@0.21.5: + resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + requiresBuild: true + dev: false + optional: true + /@esbuild/android-x64@0.16.17: resolution: {integrity: sha512-a3kTv3m0Ghh4z1DaFEuEDfz3OLONKuFvI4Xqczqx4BqLyuFaFkuaG4j2MtA6fuWEFeC5x9IvqnX7drmRq/fyAQ==} engines: {node: '>=12'} @@ -2492,6 +2564,15 @@ packages: requiresBuild: true optional: true + /@esbuild/android-x64@0.21.5: + resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + requiresBuild: true + dev: false + optional: true + /@esbuild/darwin-arm64@0.16.17: resolution: {integrity: sha512-/2agbUEfmxWHi9ARTX6OQ/KgXnOWfsNlTeLcoV7HSuSTv63E4DqtAc+2XqGw1KHxKMHGZgbVCZge7HXWX9Vn+w==} engines: {node: '>=12'} @@ -2509,6 +2590,15 @@ packages: requiresBuild: true optional: true + /@esbuild/darwin-arm64@0.21.5: + resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + /@esbuild/darwin-x64@0.16.17: resolution: {integrity: sha512-2By45OBHulkd9Svy5IOCZt376Aa2oOkiE9QWUK9fe6Tb+WDr8hXL3dpqi+DeLiMed8tVXspzsTAvd0jUl96wmg==} engines: {node: '>=12'} @@ -2526,6 +2616,15 @@ packages: requiresBuild: true optional: true + /@esbuild/darwin-x64@0.21.5: + resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + /@esbuild/freebsd-arm64@0.16.17: resolution: {integrity: sha512-mt+cxZe1tVx489VTb4mBAOo2aKSnJ33L9fr25JXpqQqzbUIw/yzIzi+NHwAXK2qYV1lEFp4OoVeThGjUbmWmdw==} engines: {node: '>=12'} @@ -2543,6 +2642,15 @@ packages: requiresBuild: true optional: true + /@esbuild/freebsd-arm64@0.21.5: + resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + requiresBuild: true + dev: false + optional: true + /@esbuild/freebsd-x64@0.16.17: resolution: {integrity: sha512-8ScTdNJl5idAKjH8zGAsN7RuWcyHG3BAvMNpKOBaqqR7EbUhhVHOqXRdL7oZvz8WNHL2pr5+eIT5c65kA6NHug==} engines: {node: '>=12'} @@ -2560,6 +2668,15 @@ packages: requiresBuild: true optional: true + /@esbuild/freebsd-x64@0.21.5: + resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + requiresBuild: true + dev: false + optional: true + /@esbuild/linux-arm64@0.16.17: resolution: {integrity: sha512-7S8gJnSlqKGVJunnMCrXHU9Q8Q/tQIxk/xL8BqAP64wchPCTzuM6W3Ra8cIa1HIflAvDnNOt2jaL17vaW+1V0g==} engines: {node: '>=12'} @@ -2577,6 +2694,15 @@ packages: requiresBuild: true optional: true + /@esbuild/linux-arm64@0.21.5: + resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@esbuild/linux-arm@0.16.17: resolution: {integrity: sha512-iihzrWbD4gIT7j3caMzKb/RsFFHCwqqbrbH9SqUSRrdXkXaygSZCZg1FybsZz57Ju7N/SHEgPyaR0LZ8Zbe9gQ==} engines: {node: '>=12'} @@ -2594,6 +2720,15 @@ packages: requiresBuild: true optional: true + /@esbuild/linux-arm@0.21.5: + resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@esbuild/linux-ia32@0.16.17: resolution: {integrity: sha512-kiX69+wcPAdgl3Lonh1VI7MBr16nktEvOfViszBSxygRQqSpzv7BffMKRPMFwzeJGPxcio0pdD3kYQGpqQ2SSg==} engines: {node: '>=12'} @@ -2611,6 +2746,15 @@ packages: requiresBuild: true optional: true + /@esbuild/linux-ia32@0.21.5: + resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@esbuild/linux-loong64@0.16.17: resolution: {integrity: sha512-dTzNnQwembNDhd654cA4QhbS9uDdXC3TKqMJjgOWsC0yNCbpzfWoXdZvp0mY7HU6nzk5E0zpRGGx3qoQg8T2DQ==} engines: {node: '>=12'} @@ -2628,6 +2772,15 @@ packages: requiresBuild: true optional: true + /@esbuild/linux-loong64@0.21.5: + resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@esbuild/linux-mips64el@0.16.17: resolution: {integrity: sha512-ezbDkp2nDl0PfIUn0CsQ30kxfcLTlcx4Foz2kYv8qdC6ia2oX5Q3E/8m6lq84Dj/6b0FrkgD582fJMIfHhJfSw==} engines: {node: '>=12'} @@ -2645,6 +2798,15 @@ packages: requiresBuild: true optional: true + /@esbuild/linux-mips64el@0.21.5: + resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@esbuild/linux-ppc64@0.16.17: resolution: {integrity: sha512-dzS678gYD1lJsW73zrFhDApLVdM3cUF2MvAa1D8K8KtcSKdLBPP4zZSLy6LFZ0jYqQdQ29bjAHJDgz0rVbLB3g==} engines: {node: '>=12'} @@ -2662,6 +2824,15 @@ packages: requiresBuild: true optional: true + /@esbuild/linux-ppc64@0.21.5: + resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@esbuild/linux-riscv64@0.16.17: resolution: {integrity: sha512-ylNlVsxuFjZK8DQtNUwiMskh6nT0vI7kYl/4fZgV1llP5d6+HIeL/vmmm3jpuoo8+NuXjQVZxmKuhDApK0/cKw==} engines: {node: '>=12'} @@ -2679,6 +2850,15 @@ packages: requiresBuild: true optional: true + /@esbuild/linux-riscv64@0.21.5: + resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@esbuild/linux-s390x@0.16.17: resolution: {integrity: sha512-gzy7nUTO4UA4oZ2wAMXPNBGTzZFP7mss3aKR2hH+/4UUkCOyqmjXiKpzGrY2TlEUhbbejzXVKKGazYcQTZWA/w==} engines: {node: '>=12'} @@ -2696,6 +2876,15 @@ packages: requiresBuild: true optional: true + /@esbuild/linux-s390x@0.21.5: + resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@esbuild/linux-x64@0.16.17: resolution: {integrity: sha512-mdPjPxfnmoqhgpiEArqi4egmBAMYvaObgn4poorpUaqmvzzbvqbowRllQ+ZgzGVMGKaPkqUmPDOOFQRUFDmeUw==} engines: {node: '>=12'} @@ -2713,6 +2902,15 @@ packages: requiresBuild: true optional: true + /@esbuild/linux-x64@0.21.5: + resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + /@esbuild/netbsd-x64@0.16.17: resolution: {integrity: sha512-/PzmzD/zyAeTUsduZa32bn0ORug+Jd1EGGAUJvqfeixoEISYpGnAezN6lnJoskauoai0Jrs+XSyvDhppCPoKOA==} engines: {node: '>=12'} @@ -2730,6 +2928,15 @@ packages: requiresBuild: true optional: true + /@esbuild/netbsd-x64@0.21.5: + resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + requiresBuild: true + dev: false + optional: true + /@esbuild/openbsd-x64@0.16.17: resolution: {integrity: sha512-2yaWJhvxGEz2RiftSk0UObqJa/b+rIAjnODJgv2GbGGpRwAfpgzyrg1WLK8rqA24mfZa9GvpjLcBBg8JHkoodg==} engines: {node: '>=12'} @@ -2747,6 +2954,15 @@ packages: requiresBuild: true optional: true + /@esbuild/openbsd-x64@0.21.5: + resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + requiresBuild: true + dev: false + optional: true + /@esbuild/sunos-x64@0.16.17: resolution: {integrity: sha512-xtVUiev38tN0R3g8VhRfN7Zl42YCJvyBhRKw1RJjwE1d2emWTVToPLNEQj/5Qxc6lVFATDiy6LjVHYhIPrLxzw==} engines: {node: '>=12'} @@ -2764,6 +2980,15 @@ packages: requiresBuild: true optional: true + /@esbuild/sunos-x64@0.21.5: + resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + requiresBuild: true + dev: false + optional: true + /@esbuild/win32-arm64@0.16.17: resolution: {integrity: sha512-ga8+JqBDHY4b6fQAmOgtJJue36scANy4l/rL97W+0wYmijhxKetzZdKOJI7olaBaMhWt8Pac2McJdZLxXWUEQw==} engines: {node: '>=12'} @@ -2781,6 +3006,15 @@ packages: requiresBuild: true optional: true + /@esbuild/win32-arm64@0.21.5: + resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: false + optional: true + /@esbuild/win32-ia32@0.16.17: resolution: {integrity: sha512-WnsKaf46uSSF/sZhwnqE4L/F89AYNMiD4YtEcYekBt9Q7nj0DiId2XH2Ng2PHM54qi5oPrQ8luuzGszqi/veig==} engines: {node: '>=12'} @@ -2798,6 +3032,15 @@ packages: requiresBuild: true optional: true + /@esbuild/win32-ia32@0.21.5: + resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: false + optional: true + /@esbuild/win32-x64@0.16.17: resolution: {integrity: sha512-y+EHuSchhL7FjHgvQL/0fnnFmO4T1bhvWANX6gcnqTjtnKWbTvUMCpGnv2+t+31d7RzyEAYAd4u2fnIhHL6N/Q==} engines: {node: '>=12'} @@ -2815,6 +3058,15 @@ packages: requiresBuild: true optional: true + /@esbuild/win32-x64@0.21.5: + resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + /@escape.tech/graphql-armor-max-aliases@2.4.0: resolution: {integrity: sha512-d4V9EgtPRG9HIoPHuanFNLHj1ENB1YkZi9FbiBiH88x5VahCjVpMXDgKQGkG6RUTOODU4XKp0/ZgaOq0pX5oEA==} engines: {node: '>=18.0.0'} @@ -3211,6 +3463,17 @@ packages: engines: {node: '>=18.14.1'} dev: false + /@hono/trpc-server@0.3.2(@trpc/server@10.45.2)(hono@4.5.0): + resolution: {integrity: sha512-dTKDrSldjBn0hi9FjHOGdiHgMCWPoW5NaBUTQRNLyVL9JcJJu9oiwTFoIscPFxc9CF2rAeA8aRGHoFJK+A8cLw==} + engines: {node: '>=16.0.0'} + peerDependencies: + '@trpc/server': ^10.10.0 || >11.0.0-rc + hono: '>=4.*' + dependencies: + '@trpc/server': 10.45.2 + hono: 4.5.0 + dev: false + /@humanwhocodes/config-array@0.11.13: resolution: {integrity: sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ==} engines: {node: '>=10.10.0'} @@ -4187,6 +4450,16 @@ packages: unist-util-visit: 5.0.0 dev: false + /@trpc/client@10.45.2: + resolution: {integrity: sha512-ykALM5kYWTLn1zYuUOZ2cPWlVfrXhc18HzBDyRhoPYN0jey4iQHEFSEowfnhg1RvYnrAVjNBgHNeSAXjrDbGwg==} + peerDependencies: + '@trpc/server': 10.45.2 + dev: false + + /@trpc/server@10.45.2: + resolution: {integrity: sha512-wOrSThNNE4HUnuhJG6PfDRp4L2009KDVxsd+2VYH8ro6o/7/jwYZ8Uu5j+VaW+mOmc8EHerHzGcdbGNQSAUPgg==} + dev: false + /@trysound/sax@0.2.0: resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==} engines: {node: '>=10.13.0'} @@ -4743,7 +5016,7 @@ packages: zod: 3.22.4 dev: true - /abitype@0.9.8(typescript@5.3.3): + /abitype@0.9.8(typescript@5.3.3)(zod@3.23.8): resolution: {integrity: sha512-puLifILdm+8sjyss4S+fsUN09obiT1g2YW6CtcQF+QDzxR0euzgEB29MZujC6zMk2a6SVmtttq1fc6+YFA7WYQ==} peerDependencies: typescript: '>=5.0.4' @@ -4755,6 +5028,7 @@ packages: optional: true dependencies: typescript: 5.3.3 + zod: 3.23.8 /abitype@1.0.0(typescript@5.0.4): resolution: {integrity: sha512-NMeMah//6bJ56H5XRj8QCV4AwuW6hB6zqz2LnhhLdcWVQOsXki6/Pn3APeqxCma62nXIcmZWdu1DlHWS74umVQ==} @@ -7153,6 +7427,37 @@ packages: '@esbuild/win32-ia32': 0.19.11 '@esbuild/win32-x64': 0.19.11 + /esbuild@0.21.5: + resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} + engines: {node: '>=12'} + hasBin: true + requiresBuild: true + optionalDependencies: + '@esbuild/aix-ppc64': 0.21.5 + '@esbuild/android-arm': 0.21.5 + '@esbuild/android-arm64': 0.21.5 + '@esbuild/android-x64': 0.21.5 + '@esbuild/darwin-arm64': 0.21.5 + '@esbuild/darwin-x64': 0.21.5 + '@esbuild/freebsd-arm64': 0.21.5 + '@esbuild/freebsd-x64': 0.21.5 + '@esbuild/linux-arm': 0.21.5 + '@esbuild/linux-arm64': 0.21.5 + '@esbuild/linux-ia32': 0.21.5 + '@esbuild/linux-loong64': 0.21.5 + '@esbuild/linux-mips64el': 0.21.5 + '@esbuild/linux-ppc64': 0.21.5 + '@esbuild/linux-riscv64': 0.21.5 + '@esbuild/linux-s390x': 0.21.5 + '@esbuild/linux-x64': 0.21.5 + '@esbuild/netbsd-x64': 0.21.5 + '@esbuild/openbsd-x64': 0.21.5 + '@esbuild/sunos-x64': 0.21.5 + '@esbuild/win32-arm64': 0.21.5 + '@esbuild/win32-ia32': 0.21.5 + '@esbuild/win32-x64': 0.21.5 + dev: false + /escalade@3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} @@ -8102,6 +8407,12 @@ packages: resolve-pkg-maps: 1.0.0 dev: true + /get-tsconfig@4.7.5: + resolution: {integrity: sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==} + dependencies: + resolve-pkg-maps: 1.0.0 + dev: false + /getpass@0.1.7: resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} dependencies: @@ -12644,7 +12955,6 @@ packages: /resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - dev: true /resolve@1.22.8: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} @@ -13915,6 +14225,17 @@ packages: - ts-node dev: true + /tsx@4.16.2: + resolution: {integrity: sha512-C1uWweJDgdtX2x600HjaFaucXTilT7tgUZHbOE4+ypskZ1OP8CRCSDkCxG6Vya9EwaFIVagWwpaVAn5wzypaqQ==} + engines: {node: '>=18.0.0'} + hasBin: true + dependencies: + esbuild: 0.21.5 + get-tsconfig: 4.7.5 + optionalDependencies: + fsevents: 2.3.3 + dev: false + /tty-table@4.2.3: resolution: {integrity: sha512-Fs15mu0vGzCrj8fmJNP7Ynxt5J7praPXqFN0leZeZBXJwkMxv9cb2D454k1ltrtUSJbZ4yH4e0CynsHLxmUfFA==} engines: {node: '>=8.0.0'} @@ -14441,7 +14762,7 @@ packages: '@noble/hashes': 1.3.2 '@scure/bip32': 1.3.2 '@scure/bip39': 1.2.1 - abitype: 0.9.8(typescript@5.3.3) + abitype: 0.9.8(typescript@5.3.3)(zod@3.23.8) isows: 1.0.3(ws@8.13.0) typescript: 5.3.3 ws: 8.13.0 @@ -14474,7 +14795,7 @@ packages: - zod dev: true - /viem@1.21.4(typescript@5.3.3): + /viem@1.21.4(typescript@5.3.3)(zod@3.23.8): resolution: {integrity: sha512-BNVYdSaUjeS2zKQgPs+49e5JKocfo60Ib2yiXOWBT6LuVxY1I/6fFX3waEtpXvL1Xn4qu+BVitVtMh9lyThyhQ==} peerDependencies: typescript: '>=5.0.4' @@ -14487,7 +14808,7 @@ packages: '@noble/hashes': 1.3.2 '@scure/bip32': 1.3.2 '@scure/bip39': 1.2.1 - abitype: 0.9.8(typescript@5.3.3) + abitype: 0.9.8(typescript@5.3.3)(zod@3.23.8) isows: 1.0.3(ws@8.13.0) typescript: 5.3.3 ws: 8.13.0 @@ -15043,6 +15364,9 @@ packages: /zod@3.22.4: resolution: {integrity: sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==} + /zod@3.23.8: + resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} + /zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} dev: false From 1cc22a1c2f53bd24b6205e138841a79ac823d3ec Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 20:41:29 -0400 Subject: [PATCH 108/122] codegen --- packages/core/src/common/codegen.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/common/codegen.ts b/packages/core/src/common/codegen.ts index 72848ecdf..59b5d6baa 100644 --- a/packages/core/src/common/codegen.ts +++ b/packages/core/src/common/codegen.ts @@ -26,6 +26,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; From d54af6d89f0abf2d738f4e5b31f4c27f1edec665 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 20:45:45 -0400 Subject: [PATCH 109/122] cleanup new example --- .changeset/config.json | 5 ++++- examples/with-trpc/client/package.json | 9 +++++---- examples/with-trpc/ponder/ponder-env.d.ts | 1 - packages/create-ponder/tsup.config.ts | 1 + 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.changeset/config.json b/.changeset/config.json index 39874eb3c..7cf94a65f 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -22,7 +22,10 @@ "ponder-examples-with-foundry-ponder", "ponder-examples-with-foundry-foundry", "ponder-examples-with-nextjs", - "ponder-examples-with-nextjs-ponder" + "ponder-examples-with-nextjs-ponder", + "ponder-examples-with-trpc", + "ponder-examples-with-trpc-ponder", + "ponder-examples-with-trpc-client" ], "linked": [], "access": "public", diff --git a/examples/with-trpc/client/package.json b/examples/with-trpc/client/package.json index 881e96934..02f06cc8f 100644 --- a/examples/with-trpc/client/package.json +++ b/examples/with-trpc/client/package.json @@ -1,11 +1,12 @@ { + "name": "ponder-examples-with-trpc-client", "private": true, - "name": "ponder-examples-with-trpc-ponder", "type": "module", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, + "scripts": {}, "dependencies": { "@trpc/client": "^10.45.2" + }, + "engines": { + "node": ">=18.14" } } diff --git a/examples/with-trpc/ponder/ponder-env.d.ts b/examples/with-trpc/ponder/ponder-env.d.ts index a99dcf263..f8e7347cf 100644 --- a/examples/with-trpc/ponder/ponder-env.d.ts +++ b/examples/with-trpc/ponder/ponder-env.d.ts @@ -23,6 +23,5 @@ declare module "@/generated" { >; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; - export type ApiContext = Virtual.Drizzle; export type Schema = Virtual.Schema; } diff --git a/packages/create-ponder/tsup.config.ts b/packages/create-ponder/tsup.config.ts index e762cc512..5a18d583c 100644 --- a/packages/create-ponder/tsup.config.ts +++ b/packages/create-ponder/tsup.config.ts @@ -26,6 +26,7 @@ export default defineConfig({ path.join(examplesPath, "**", "*"), "!**/with-nextjs/**", "!**/with-foundry/**", + "!**/with-trpc/**", "!**/node_modules/**", "!**/generated/**", "!**/.ponder/**", From 71a56cd853989d0ccd9a5180a4a4f34dbfa84c3a Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 20:46:57 -0400 Subject: [PATCH 110/122] fix lockfile --- pnpm-lock.yaml | 262 +------------------------------------------------ 1 file changed, 1 insertion(+), 261 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5c8a527d0..4f05809d8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -553,12 +553,6 @@ importers: '@trpc/client': specifier: ^10.45.2 version: 10.45.2 - hono: - specifier: ^4.4.6 - version: 4.5.0 - tsx: - specifier: ^4.16.2 - version: 4.16.2 examples/with-trpc/ponder: dependencies: @@ -2486,15 +2480,6 @@ packages: requiresBuild: true optional: true - /@esbuild/aix-ppc64@0.21.5: - resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [aix] - requiresBuild: true - dev: false - optional: true - /@esbuild/android-arm64@0.16.17: resolution: {integrity: sha512-MIGl6p5sc3RDTLLkYL1MyL8BMRN4tLMRCn+yRJJmEDvYZ2M7tmAf80hx1kbNEUX2KJ50RRtxZ4JHLvCfuB6kBg==} engines: {node: '>=12'} @@ -2512,15 +2497,6 @@ packages: requiresBuild: true optional: true - /@esbuild/android-arm64@0.21.5: - resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} - engines: {node: '>=12'} - cpu: [arm64] - os: [android] - requiresBuild: true - dev: false - optional: true - /@esbuild/android-arm@0.16.17: resolution: {integrity: sha512-N9x1CMXVhtWEAMS7pNNONyA14f71VPQN9Cnavj1XQh6T7bskqiLLrSca4O0Vr8Wdcga943eThxnVp3JLnBMYtw==} engines: {node: '>=12'} @@ -2538,15 +2514,6 @@ packages: requiresBuild: true optional: true - /@esbuild/android-arm@0.21.5: - resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} - engines: {node: '>=12'} - cpu: [arm] - os: [android] - requiresBuild: true - dev: false - optional: true - /@esbuild/android-x64@0.16.17: resolution: {integrity: sha512-a3kTv3m0Ghh4z1DaFEuEDfz3OLONKuFvI4Xqczqx4BqLyuFaFkuaG4j2MtA6fuWEFeC5x9IvqnX7drmRq/fyAQ==} engines: {node: '>=12'} @@ -2564,15 +2531,6 @@ packages: requiresBuild: true optional: true - /@esbuild/android-x64@0.21.5: - resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} - engines: {node: '>=12'} - cpu: [x64] - os: [android] - requiresBuild: true - dev: false - optional: true - /@esbuild/darwin-arm64@0.16.17: resolution: {integrity: sha512-/2agbUEfmxWHi9ARTX6OQ/KgXnOWfsNlTeLcoV7HSuSTv63E4DqtAc+2XqGw1KHxKMHGZgbVCZge7HXWX9Vn+w==} engines: {node: '>=12'} @@ -2590,15 +2548,6 @@ packages: requiresBuild: true optional: true - /@esbuild/darwin-arm64@0.21.5: - resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} - engines: {node: '>=12'} - cpu: [arm64] - os: [darwin] - requiresBuild: true - dev: false - optional: true - /@esbuild/darwin-x64@0.16.17: resolution: {integrity: sha512-2By45OBHulkd9Svy5IOCZt376Aa2oOkiE9QWUK9fe6Tb+WDr8hXL3dpqi+DeLiMed8tVXspzsTAvd0jUl96wmg==} engines: {node: '>=12'} @@ -2616,15 +2565,6 @@ packages: requiresBuild: true optional: true - /@esbuild/darwin-x64@0.21.5: - resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} - engines: {node: '>=12'} - cpu: [x64] - os: [darwin] - requiresBuild: true - dev: false - optional: true - /@esbuild/freebsd-arm64@0.16.17: resolution: {integrity: sha512-mt+cxZe1tVx489VTb4mBAOo2aKSnJ33L9fr25JXpqQqzbUIw/yzIzi+NHwAXK2qYV1lEFp4OoVeThGjUbmWmdw==} engines: {node: '>=12'} @@ -2642,15 +2582,6 @@ packages: requiresBuild: true optional: true - /@esbuild/freebsd-arm64@0.21.5: - resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} - engines: {node: '>=12'} - cpu: [arm64] - os: [freebsd] - requiresBuild: true - dev: false - optional: true - /@esbuild/freebsd-x64@0.16.17: resolution: {integrity: sha512-8ScTdNJl5idAKjH8zGAsN7RuWcyHG3BAvMNpKOBaqqR7EbUhhVHOqXRdL7oZvz8WNHL2pr5+eIT5c65kA6NHug==} engines: {node: '>=12'} @@ -2668,15 +2599,6 @@ packages: requiresBuild: true optional: true - /@esbuild/freebsd-x64@0.21.5: - resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [freebsd] - requiresBuild: true - dev: false - optional: true - /@esbuild/linux-arm64@0.16.17: resolution: {integrity: sha512-7S8gJnSlqKGVJunnMCrXHU9Q8Q/tQIxk/xL8BqAP64wchPCTzuM6W3Ra8cIa1HIflAvDnNOt2jaL17vaW+1V0g==} engines: {node: '>=12'} @@ -2694,15 +2616,6 @@ packages: requiresBuild: true optional: true - /@esbuild/linux-arm64@0.21.5: - resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} - engines: {node: '>=12'} - cpu: [arm64] - os: [linux] - requiresBuild: true - dev: false - optional: true - /@esbuild/linux-arm@0.16.17: resolution: {integrity: sha512-iihzrWbD4gIT7j3caMzKb/RsFFHCwqqbrbH9SqUSRrdXkXaygSZCZg1FybsZz57Ju7N/SHEgPyaR0LZ8Zbe9gQ==} engines: {node: '>=12'} @@ -2720,15 +2633,6 @@ packages: requiresBuild: true optional: true - /@esbuild/linux-arm@0.21.5: - resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} - engines: {node: '>=12'} - cpu: [arm] - os: [linux] - requiresBuild: true - dev: false - optional: true - /@esbuild/linux-ia32@0.16.17: resolution: {integrity: sha512-kiX69+wcPAdgl3Lonh1VI7MBr16nktEvOfViszBSxygRQqSpzv7BffMKRPMFwzeJGPxcio0pdD3kYQGpqQ2SSg==} engines: {node: '>=12'} @@ -2746,15 +2650,6 @@ packages: requiresBuild: true optional: true - /@esbuild/linux-ia32@0.21.5: - resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} - engines: {node: '>=12'} - cpu: [ia32] - os: [linux] - requiresBuild: true - dev: false - optional: true - /@esbuild/linux-loong64@0.16.17: resolution: {integrity: sha512-dTzNnQwembNDhd654cA4QhbS9uDdXC3TKqMJjgOWsC0yNCbpzfWoXdZvp0mY7HU6nzk5E0zpRGGx3qoQg8T2DQ==} engines: {node: '>=12'} @@ -2772,15 +2667,6 @@ packages: requiresBuild: true optional: true - /@esbuild/linux-loong64@0.21.5: - resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} - engines: {node: '>=12'} - cpu: [loong64] - os: [linux] - requiresBuild: true - dev: false - optional: true - /@esbuild/linux-mips64el@0.16.17: resolution: {integrity: sha512-ezbDkp2nDl0PfIUn0CsQ30kxfcLTlcx4Foz2kYv8qdC6ia2oX5Q3E/8m6lq84Dj/6b0FrkgD582fJMIfHhJfSw==} engines: {node: '>=12'} @@ -2798,15 +2684,6 @@ packages: requiresBuild: true optional: true - /@esbuild/linux-mips64el@0.21.5: - resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} - engines: {node: '>=12'} - cpu: [mips64el] - os: [linux] - requiresBuild: true - dev: false - optional: true - /@esbuild/linux-ppc64@0.16.17: resolution: {integrity: sha512-dzS678gYD1lJsW73zrFhDApLVdM3cUF2MvAa1D8K8KtcSKdLBPP4zZSLy6LFZ0jYqQdQ29bjAHJDgz0rVbLB3g==} engines: {node: '>=12'} @@ -2824,15 +2701,6 @@ packages: requiresBuild: true optional: true - /@esbuild/linux-ppc64@0.21.5: - resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [linux] - requiresBuild: true - dev: false - optional: true - /@esbuild/linux-riscv64@0.16.17: resolution: {integrity: sha512-ylNlVsxuFjZK8DQtNUwiMskh6nT0vI7kYl/4fZgV1llP5d6+HIeL/vmmm3jpuoo8+NuXjQVZxmKuhDApK0/cKw==} engines: {node: '>=12'} @@ -2850,15 +2718,6 @@ packages: requiresBuild: true optional: true - /@esbuild/linux-riscv64@0.21.5: - resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} - engines: {node: '>=12'} - cpu: [riscv64] - os: [linux] - requiresBuild: true - dev: false - optional: true - /@esbuild/linux-s390x@0.16.17: resolution: {integrity: sha512-gzy7nUTO4UA4oZ2wAMXPNBGTzZFP7mss3aKR2hH+/4UUkCOyqmjXiKpzGrY2TlEUhbbejzXVKKGazYcQTZWA/w==} engines: {node: '>=12'} @@ -2876,15 +2735,6 @@ packages: requiresBuild: true optional: true - /@esbuild/linux-s390x@0.21.5: - resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} - engines: {node: '>=12'} - cpu: [s390x] - os: [linux] - requiresBuild: true - dev: false - optional: true - /@esbuild/linux-x64@0.16.17: resolution: {integrity: sha512-mdPjPxfnmoqhgpiEArqi4egmBAMYvaObgn4poorpUaqmvzzbvqbowRllQ+ZgzGVMGKaPkqUmPDOOFQRUFDmeUw==} engines: {node: '>=12'} @@ -2902,15 +2752,6 @@ packages: requiresBuild: true optional: true - /@esbuild/linux-x64@0.21.5: - resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [linux] - requiresBuild: true - dev: false - optional: true - /@esbuild/netbsd-x64@0.16.17: resolution: {integrity: sha512-/PzmzD/zyAeTUsduZa32bn0ORug+Jd1EGGAUJvqfeixoEISYpGnAezN6lnJoskauoai0Jrs+XSyvDhppCPoKOA==} engines: {node: '>=12'} @@ -2928,15 +2769,6 @@ packages: requiresBuild: true optional: true - /@esbuild/netbsd-x64@0.21.5: - resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} - engines: {node: '>=12'} - cpu: [x64] - os: [netbsd] - requiresBuild: true - dev: false - optional: true - /@esbuild/openbsd-x64@0.16.17: resolution: {integrity: sha512-2yaWJhvxGEz2RiftSk0UObqJa/b+rIAjnODJgv2GbGGpRwAfpgzyrg1WLK8rqA24mfZa9GvpjLcBBg8JHkoodg==} engines: {node: '>=12'} @@ -2954,15 +2786,6 @@ packages: requiresBuild: true optional: true - /@esbuild/openbsd-x64@0.21.5: - resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} - engines: {node: '>=12'} - cpu: [x64] - os: [openbsd] - requiresBuild: true - dev: false - optional: true - /@esbuild/sunos-x64@0.16.17: resolution: {integrity: sha512-xtVUiev38tN0R3g8VhRfN7Zl42YCJvyBhRKw1RJjwE1d2emWTVToPLNEQj/5Qxc6lVFATDiy6LjVHYhIPrLxzw==} engines: {node: '>=12'} @@ -2980,15 +2803,6 @@ packages: requiresBuild: true optional: true - /@esbuild/sunos-x64@0.21.5: - resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} - engines: {node: '>=12'} - cpu: [x64] - os: [sunos] - requiresBuild: true - dev: false - optional: true - /@esbuild/win32-arm64@0.16.17: resolution: {integrity: sha512-ga8+JqBDHY4b6fQAmOgtJJue36scANy4l/rL97W+0wYmijhxKetzZdKOJI7olaBaMhWt8Pac2McJdZLxXWUEQw==} engines: {node: '>=12'} @@ -3006,15 +2820,6 @@ packages: requiresBuild: true optional: true - /@esbuild/win32-arm64@0.21.5: - resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} - engines: {node: '>=12'} - cpu: [arm64] - os: [win32] - requiresBuild: true - dev: false - optional: true - /@esbuild/win32-ia32@0.16.17: resolution: {integrity: sha512-WnsKaf46uSSF/sZhwnqE4L/F89AYNMiD4YtEcYekBt9Q7nj0DiId2XH2Ng2PHM54qi5oPrQ8luuzGszqi/veig==} engines: {node: '>=12'} @@ -3032,15 +2837,6 @@ packages: requiresBuild: true optional: true - /@esbuild/win32-ia32@0.21.5: - resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} - engines: {node: '>=12'} - cpu: [ia32] - os: [win32] - requiresBuild: true - dev: false - optional: true - /@esbuild/win32-x64@0.16.17: resolution: {integrity: sha512-y+EHuSchhL7FjHgvQL/0fnnFmO4T1bhvWANX6gcnqTjtnKWbTvUMCpGnv2+t+31d7RzyEAYAd4u2fnIhHL6N/Q==} engines: {node: '>=12'} @@ -3058,15 +2854,6 @@ packages: requiresBuild: true optional: true - /@esbuild/win32-x64@0.21.5: - resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} - engines: {node: '>=12'} - cpu: [x64] - os: [win32] - requiresBuild: true - dev: false - optional: true - /@escape.tech/graphql-armor-max-aliases@2.4.0: resolution: {integrity: sha512-d4V9EgtPRG9HIoPHuanFNLHj1ENB1YkZi9FbiBiH88x5VahCjVpMXDgKQGkG6RUTOODU4XKp0/ZgaOq0pX5oEA==} engines: {node: '>=18.0.0'} @@ -7427,37 +7214,6 @@ packages: '@esbuild/win32-ia32': 0.19.11 '@esbuild/win32-x64': 0.19.11 - /esbuild@0.21.5: - resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} - engines: {node: '>=12'} - hasBin: true - requiresBuild: true - optionalDependencies: - '@esbuild/aix-ppc64': 0.21.5 - '@esbuild/android-arm': 0.21.5 - '@esbuild/android-arm64': 0.21.5 - '@esbuild/android-x64': 0.21.5 - '@esbuild/darwin-arm64': 0.21.5 - '@esbuild/darwin-x64': 0.21.5 - '@esbuild/freebsd-arm64': 0.21.5 - '@esbuild/freebsd-x64': 0.21.5 - '@esbuild/linux-arm': 0.21.5 - '@esbuild/linux-arm64': 0.21.5 - '@esbuild/linux-ia32': 0.21.5 - '@esbuild/linux-loong64': 0.21.5 - '@esbuild/linux-mips64el': 0.21.5 - '@esbuild/linux-ppc64': 0.21.5 - '@esbuild/linux-riscv64': 0.21.5 - '@esbuild/linux-s390x': 0.21.5 - '@esbuild/linux-x64': 0.21.5 - '@esbuild/netbsd-x64': 0.21.5 - '@esbuild/openbsd-x64': 0.21.5 - '@esbuild/sunos-x64': 0.21.5 - '@esbuild/win32-arm64': 0.21.5 - '@esbuild/win32-ia32': 0.21.5 - '@esbuild/win32-x64': 0.21.5 - dev: false - /escalade@3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} @@ -8407,12 +8163,6 @@ packages: resolve-pkg-maps: 1.0.0 dev: true - /get-tsconfig@4.7.5: - resolution: {integrity: sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==} - dependencies: - resolve-pkg-maps: 1.0.0 - dev: false - /getpass@0.1.7: resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} dependencies: @@ -12955,6 +12705,7 @@ packages: /resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + dev: true /resolve@1.22.8: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} @@ -14225,17 +13976,6 @@ packages: - ts-node dev: true - /tsx@4.16.2: - resolution: {integrity: sha512-C1uWweJDgdtX2x600HjaFaucXTilT7tgUZHbOE4+ypskZ1OP8CRCSDkCxG6Vya9EwaFIVagWwpaVAn5wzypaqQ==} - engines: {node: '>=18.0.0'} - hasBin: true - dependencies: - esbuild: 0.21.5 - get-tsconfig: 4.7.5 - optionalDependencies: - fsevents: 2.3.3 - dev: false - /tty-table@4.2.3: resolution: {integrity: sha512-Fs15mu0vGzCrj8fmJNP7Ynxt5J7praPXqFN0leZeZBXJwkMxv9cb2D454k1ltrtUSJbZ4yH4e0CynsHLxmUfFA==} engines: {node: '>=8.0.0'} From 757f6016b5c4e914145ac64c9d160104b575e1f9 Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Tue, 16 Jul 2024 20:49:06 -0400 Subject: [PATCH 111/122] readonly database --- .../core/src/database/postgres/service.ts | 15 ++------- packages/core/src/database/sqlite/service.ts | 20 ++++++++---- packages/core/src/drizzle/runtime.ts | 28 +++++++++++++--- packages/core/src/utils/pg.ts | 32 +++++++++++++++++++ packages/core/src/utils/sqlite.ts | 11 +++++++ 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/packages/core/src/database/postgres/service.ts b/packages/core/src/database/postgres/service.ts index 78522033c..352077f42 100644 --- a/packages/core/src/database/postgres/service.ts +++ b/packages/core/src/database/postgres/service.ts @@ -26,7 +26,7 @@ import { } from "@/utils/checkpoint.js"; import { formatEta } from "@/utils/format.js"; import { hash } from "@/utils/hash.js"; -import { createPool } from "@/utils/pg.js"; +import { createPool, createReadonlyPool } from "@/utils/pg.js"; import { wait } from "@/utils/wait.js"; import { type CreateTableBuilder, @@ -108,7 +108,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { application_name: `${userNamespace}_indexing`, max: indexingMax, }); - this.readonlyPool = createPool({ + this.readonlyPool = createReadonlyPool({ ...poolConfig, application_name: `${userNamespace}_readonly`, max: syncMax, @@ -151,16 +151,7 @@ export class PostgresDatabaseService implements BaseDatabaseService { this.readonlyDb = new HeadlessKysely({ name: "readonly", common, - dialect: new PostgresDialect({ - pool: this.readonlyPool, - onCreateConnection: async (connection) => { - await connection.executeQuery( - sql - .raw("SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY") - .compile(this.readonlyDb), - ); - }, - }), + dialect: new PostgresDialect({ pool: this.readonlyPool }), log(event) { if (event.level === "query") { common.metrics.ponder_postgres_query_total.inc({ pool: "readonly" }); diff --git a/packages/core/src/database/sqlite/service.ts b/packages/core/src/database/sqlite/service.ts index 7b0c05160..8fd82d0a9 100644 --- a/packages/core/src/database/sqlite/service.ts +++ b/packages/core/src/database/sqlite/service.ts @@ -24,7 +24,11 @@ import { } from "@/utils/checkpoint.js"; import { formatEta } from "@/utils/format.js"; import { hash } from "@/utils/hash.js"; -import { type SqliteDatabase, createSqliteDatabase } from "@/utils/sqlite.js"; +import { + type SqliteDatabase, + createReadonlySqliteDatabase, + createSqliteDatabase, +} from "@/utils/sqlite.js"; import { wait } from "@/utils/wait.js"; import { type CreateTableBuilder, @@ -51,10 +55,9 @@ export class SqliteDatabaseService implements BaseDatabaseService { private userNamespace: string; private internalNamespace: string; - userDatabaseFile: string; - private internalDatabase: SqliteDatabase; private syncDatabase: SqliteDatabase; + readonlyDatabase: SqliteDatabase; db: HeadlessKysely; readonlyDb: HeadlessKysely; @@ -79,7 +82,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { this.deleteV3DatabaseFiles(); this.userNamespace = userNamespace; - this.userDatabaseFile = path.join(directory, `${userNamespace}.db`); + const userDatabaseFile = path.join(directory, `${userNamespace}.db`); // Note that SQLite supports using "main" as the schema name for tables // in the primary database (as opposed to attached databases). We include @@ -90,7 +93,12 @@ export class SqliteDatabaseService implements BaseDatabaseService { this.internalDatabase = createSqliteDatabase(internalDatabaseFile); this.internalDatabase.exec( - `ATTACH DATABASE '${this.userDatabaseFile}' AS ${this.userNamespace}`, + `ATTACH DATABASE '${userDatabaseFile}' AS ${this.userNamespace}`, + ); + + this.readonlyDatabase = createReadonlySqliteDatabase(internalDatabaseFile); + this.readonlyDatabase.exec( + `ATTACH DATABASE '${userDatabaseFile}' AS ${this.userNamespace}`, ); this.db = new HeadlessKysely({ @@ -135,7 +143,7 @@ export class SqliteDatabaseService implements BaseDatabaseService { this.readonlyDb = new HeadlessKysely({ name: "readonly", common, - dialect: new SqliteDialect({ database: this.internalDatabase }), + dialect: new SqliteDialect({ database: this.readonlyDatabase }), log(event) { if (event.level === "query") { common.metrics.ponder_sqlite_query_total.inc({ diff --git a/packages/core/src/drizzle/runtime.ts b/packages/core/src/drizzle/runtime.ts index 763949a51..33369d35a 100644 --- a/packages/core/src/drizzle/runtime.ts +++ b/packages/core/src/drizzle/runtime.ts @@ -10,7 +10,6 @@ import { isScalarColumn, } from "@/schema/utils.js"; import { getTables } from "@/schema/utils.js"; -import { createSqliteDatabase } from "@/utils/sqlite.js"; import { drizzle as drizzleSQLite } from "drizzle-orm/better-sqlite3"; import { drizzle as drizzlePg } from "drizzle-orm/node-postgres"; import { pgSchema, pgTable } from "drizzle-orm/pg-core"; @@ -41,13 +40,32 @@ export const createDrizzleDb = (database: DatabaseService) => { execute: (query: any) => drizzle.execute(query), }; } else { - const drizzle = drizzleSQLite( - createSqliteDatabase(database.userDatabaseFile), - ); + const drizzle = drizzleSQLite(database.readonlyDatabase); return { // @ts-ignore select: (...args: any[]) => drizzle.select(...args), - execute: (query: any) => drizzle.all(query), + execute: (query: any) => { + try { + try { + return drizzle.all(query); + } catch (e) { + const error = e as Error; + if ( + error.name === "SqliteError" && + error.message === + "This statement does not return data. Use run() instead" + ) { + return drizzle.run(query); + } else { + throw error; + } + } + } catch (e) { + const error = e as Error; + if (error.cause) throw error.cause; + throw error; + } + }, }; } }; diff --git a/packages/core/src/utils/pg.ts b/packages/core/src/utils/pg.ts index 3e2ef72d5..2d8664d7b 100644 --- a/packages/core/src/utils/pg.ts +++ b/packages/core/src/utils/pg.ts @@ -44,6 +44,28 @@ pg.Client.prototype.query = function query( } }; +class ReadonlyClient extends pg.Client { + // @ts-expect-error + override connect( + callback: (err: Error) => void | undefined, + ): void | Promise { + if (callback) { + super.connect(() => { + this.query( + "SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY", + callback, + ); + }); + } else { + return super.connect().then(async () => { + await this.query( + "SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY", + ); + }); + } + } +} + export function createPool(config: PoolConfig) { return new pg.Pool({ // https://stackoverflow.com/questions/59155572/how-to-set-query-timeout-in-relation-to-statement-timeout @@ -51,3 +73,13 @@ export function createPool(config: PoolConfig) { ...config, }); } + +export function createReadonlyPool(config: PoolConfig) { + return new pg.Pool({ + // https://stackoverflow.com/questions/59155572/how-to-set-query-timeout-in-relation-to-statement-timeout + statement_timeout: 2 * 60 * 1000, // 2 minutes + // @ts-expect-error: The custom Client is an undocumented option. + Client: ReadonlyClient, + ...config, + }); +} diff --git a/packages/core/src/utils/sqlite.ts b/packages/core/src/utils/sqlite.ts index c1f48fb1d..7a9280f2e 100644 --- a/packages/core/src/utils/sqlite.ts +++ b/packages/core/src/utils/sqlite.ts @@ -75,3 +75,14 @@ export function createSqliteDatabase( database.pragma("journal_mode = WAL"); return database; } + +export function createReadonlySqliteDatabase( + file: string, + options?: BetterSqlite3.Options, +): SqliteDatabase { + ensureDirExists(file); + const database = new BetterSqlite3(file, { readonly: true, ...options }); + improveSqliteErrors(database); + database.pragma("journal_mode = WAL"); + return database; +} From ecf4d7cf6c7b10064fc16ddc6aa78766da4f971a Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 21:06:04 -0400 Subject: [PATCH 112/122] fix: dev server reloading --- packages/core/src/bin/commands/dev.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/core/src/bin/commands/dev.ts b/packages/core/src/bin/commands/dev.ts index 21e5f23b4..249bbcf98 100644 --- a/packages/core/src/bin/commands/dev.ts +++ b/packages/core/src/bin/commands/dev.ts @@ -106,9 +106,6 @@ export async function dev({ cliOptions }: { cliOptions: CliOptions }) { await apiCleanupReloadable(); if (result.status === "success") { - uiService.reset(); - metrics.resetMetrics(); - apiCleanupReloadable = await runServer({ common, build: result.build, From dfe2a78eb2bbd585e298ef119fa009785f094106 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 16 Jul 2024 22:06:01 -0400 Subject: [PATCH 113/122] e2e typesafe client --- examples/with-trpc/client/env.d.ts | 1 + examples/with-trpc/client/index.ts | 1 + examples/with-trpc/ponder/ponder-env.d.ts | 1 + examples/with-trpc/ponder/src/api/index.ts | 5 +++-- 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 examples/with-trpc/client/env.d.ts diff --git a/examples/with-trpc/client/env.d.ts b/examples/with-trpc/client/env.d.ts new file mode 100644 index 000000000..74e33e49d --- /dev/null +++ b/examples/with-trpc/client/env.d.ts @@ -0,0 +1 @@ +/// diff --git a/examples/with-trpc/client/index.ts b/examples/with-trpc/client/index.ts index 3fa7d7d6e..2bd265dcc 100644 --- a/examples/with-trpc/client/index.ts +++ b/examples/with-trpc/client/index.ts @@ -11,5 +11,6 @@ const client = createTRPCProxyClient({ }); const response = await client.hello.query(zeroAddress); +// ^? console.log(response); diff --git a/examples/with-trpc/ponder/ponder-env.d.ts b/examples/with-trpc/ponder/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/with-trpc/ponder/ponder-env.d.ts +++ b/examples/with-trpc/ponder/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/with-trpc/ponder/src/api/index.ts b/examples/with-trpc/ponder/src/api/index.ts index d8babe0e9..709808d87 100644 --- a/examples/with-trpc/ponder/src/api/index.ts +++ b/examples/with-trpc/ponder/src/api/index.ts @@ -17,7 +17,8 @@ const appRouter = t.router({ .where(eq(Account.id, input as Address)) .limit(1); - return account[0]?.balance?.toString() ?? null; + if (account.length === 0) return null; + return account[0]!.balance.toString(); }), }); @@ -27,6 +28,6 @@ ponder.use( "/trpc/*", trpcServer({ router: appRouter, - createContext: (_, c) => ({ db: c.env.db, tables: c.env.tables }), + createContext: (_, c) => c.var, }), ); From 411434ad0eeed5209d4370ebb8ea661f4a3359db Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 17 Jul 2024 11:09:40 -0400 Subject: [PATCH 114/122] clean up example app --- examples/with-trpc/client/index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/with-trpc/client/index.ts b/examples/with-trpc/client/index.ts index 2bd265dcc..df33d8e8e 100644 --- a/examples/with-trpc/client/index.ts +++ b/examples/with-trpc/client/index.ts @@ -1,5 +1,4 @@ import { createTRPCProxyClient, httpBatchLink } from "@trpc/client"; -import { zeroAddress } from "viem"; import type { AppRouter } from "../ponder/src/api/index"; const client = createTRPCProxyClient({ @@ -10,7 +9,9 @@ const client = createTRPCProxyClient({ ], }); -const response = await client.hello.query(zeroAddress); -// ^? +const response = await client.hello.query( + // ^? + "0xC1894e6a52c4C7Ac5b2e0b25583Ea48bf45DA14a", +); console.log(response); From c8c49e61872fcfc6d07318d2cc6e775660c37e79 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 17 Jul 2024 11:36:08 -0400 Subject: [PATCH 115/122] codegen and server tests --- examples/feature-blocks/ponder-env.d.ts | 1 + examples/feature-call-traces/ponder-env.d.ts | 1 + examples/feature-factory/ponder-env.d.ts | 1 + examples/feature-filter/ponder-env.d.ts | 1 + examples/feature-multichain/ponder-env.d.ts | 1 + examples/feature-proxy/ponder-env.d.ts | 1 + .../feature-read-contract/ponder-env.d.ts | 1 + examples/project-friendtech/ponder-env.d.ts | 1 + .../project-uniswap-v3-flash/ponder-env.d.ts | 1 + examples/reference-erc1155/ponder-env.d.ts | 1 + examples/reference-erc20/ponder-env.d.ts | 1 + examples/reference-erc4626/ponder-env.d.ts | 1 + examples/reference-erc721/ponder-env.d.ts | 1 + examples/with-foundry/ponder/ponder-env.d.ts | 1 + examples/with-nextjs/ponder/ponder-env.d.ts | 1 + packages/core/src/build/service.ts | 23 ++---- packages/core/src/server/service.test.ts | 73 ++++++++++++++++++- packages/core/src/server/service.ts | 16 ++-- 18 files changed, 100 insertions(+), 27 deletions(-) diff --git a/examples/feature-blocks/ponder-env.d.ts b/examples/feature-blocks/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/feature-blocks/ponder-env.d.ts +++ b/examples/feature-blocks/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/feature-call-traces/ponder-env.d.ts b/examples/feature-call-traces/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/feature-call-traces/ponder-env.d.ts +++ b/examples/feature-call-traces/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/feature-factory/ponder-env.d.ts b/examples/feature-factory/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/feature-factory/ponder-env.d.ts +++ b/examples/feature-factory/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/feature-filter/ponder-env.d.ts b/examples/feature-filter/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/feature-filter/ponder-env.d.ts +++ b/examples/feature-filter/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/feature-multichain/ponder-env.d.ts b/examples/feature-multichain/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/feature-multichain/ponder-env.d.ts +++ b/examples/feature-multichain/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/feature-proxy/ponder-env.d.ts b/examples/feature-proxy/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/feature-proxy/ponder-env.d.ts +++ b/examples/feature-proxy/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/feature-read-contract/ponder-env.d.ts b/examples/feature-read-contract/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/feature-read-contract/ponder-env.d.ts +++ b/examples/feature-read-contract/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/project-friendtech/ponder-env.d.ts b/examples/project-friendtech/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/project-friendtech/ponder-env.d.ts +++ b/examples/project-friendtech/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/project-uniswap-v3-flash/ponder-env.d.ts b/examples/project-uniswap-v3-flash/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/project-uniswap-v3-flash/ponder-env.d.ts +++ b/examples/project-uniswap-v3-flash/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/reference-erc1155/ponder-env.d.ts b/examples/reference-erc1155/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/reference-erc1155/ponder-env.d.ts +++ b/examples/reference-erc1155/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/reference-erc20/ponder-env.d.ts b/examples/reference-erc20/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/reference-erc20/ponder-env.d.ts +++ b/examples/reference-erc20/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/reference-erc4626/ponder-env.d.ts b/examples/reference-erc4626/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/reference-erc4626/ponder-env.d.ts +++ b/examples/reference-erc4626/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/reference-erc721/ponder-env.d.ts b/examples/reference-erc721/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/reference-erc721/ponder-env.d.ts +++ b/examples/reference-erc721/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/with-foundry/ponder/ponder-env.d.ts b/examples/with-foundry/ponder/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/with-foundry/ponder/ponder-env.d.ts +++ b/examples/with-foundry/ponder/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/examples/with-nextjs/ponder/ponder-env.d.ts b/examples/with-nextjs/ponder/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/examples/with-nextjs/ponder/ponder-env.d.ts +++ b/examples/with-nextjs/ponder/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/packages/core/src/build/service.ts b/packages/core/src/build/service.ts index a3ce3de44..aa02adec3 100644 --- a/packages/core/src/build/service.ts +++ b/packages/core/src/build/service.ts @@ -62,8 +62,8 @@ export type IndexingBuild = BaseBuild & { }; export type ApiBuild = BaseBuild & { - app?: Hono; - routes?: PonderRoutes; + app: Hono; + routes: PonderRoutes; }; export type IndexingBuildResult = @@ -559,20 +559,11 @@ const executeApiRoutes = async ( ): Promise< | { status: "success"; - app?: Hono; - routes?: PonderRoutes; + app: Hono; + routes: PonderRoutes; } | { status: "error"; error: Error } > => { - const doesServerExist = fs.existsSync(buildService.common.options.apiDir); - const hasTsFileInServer = fs - .readdirSync(buildService.common.options.apiDir) - .some((file) => file.endsWith(".ts")); - - if (doesServerExist === false || hasTsFileInServer === false) { - return { status: "success" }; - } - const files = glob.sync(buildService.apiPattern); const executeResults = await Promise.all( files.map(async (file) => ({ @@ -687,12 +678,8 @@ const validateAndBuild = async ( const validateAndBuildApi = ( { common }: Pick, baseBuild: BaseBuild, - api: { app?: Hono; routes?: PonderRoutes }, + api: { app: Hono; routes: PonderRoutes }, ): ApiBuildResult => { - if (!api.app || !api.routes) { - return { status: "success", build: baseBuild }; - } - for (const { pathOrHandlers: [maybePathOrHandler], } of api.routes) { diff --git a/packages/core/src/server/service.test.ts b/packages/core/src/server/service.test.ts index f55786087..fd807e25f 100644 --- a/packages/core/src/server/service.test.ts +++ b/packages/core/src/server/service.test.ts @@ -3,8 +3,9 @@ import { setupDatabaseServices, setupIsolatedDatabase, } from "@/_test/setup.js"; +import type { Context } from "@/hono/context.js"; import { getMetadataStore } from "@/indexing-store/metadata.js"; -import type { Schema } from "@/schema/common.js"; +import { Hono } from "hono"; import { beforeEach, expect, test, vi } from "vitest"; import { createServer } from "./service.js"; @@ -17,6 +18,8 @@ test("port", async (context) => { const server1 = await createServer({ common: context.common, + app: new Hono(), + routes: [], schema: {}, database, dbNamespace: namespaceInfo.userNamespace, @@ -24,6 +27,8 @@ test("port", async (context) => { const server2 = await createServer({ common: context.common, + app: new Hono(), + routes: [], schema: {}, database, dbNamespace: namespaceInfo.userNamespace, @@ -42,6 +47,8 @@ test("not healthy", async (context) => { const server = await createServer({ common: context.common, + app: new Hono(), + routes: [], schema: {}, database, dbNamespace: namespaceInfo.userNamespace, @@ -61,6 +68,8 @@ test("healthy", async (context) => { const server = await createServer({ common: context.common, + app: new Hono(), + routes: [], schema: {}, database, dbNamespace: namespaceInfo.userNamespace, @@ -89,7 +98,9 @@ test("healthy PUT", async (context) => { ...context.common, options: { ...context.common.options, maxHealthcheckDuration: 0 }, }, - schema: {} as Schema, + app: new Hono(), + routes: [], + schema: {}, database, dbNamespace: namespaceInfo.userNamespace, }); @@ -110,6 +121,8 @@ test("metrics", async (context) => { const server = await createServer({ common: context.common, + app: new Hono(), + routes: [], schema: {}, database, dbNamespace: namespaceInfo.userNamespace, @@ -129,6 +142,8 @@ test("metrics error", async (context) => { const server = await createServer({ common: context.common, + app: new Hono(), + routes: [], schema: {}, database, dbNamespace: namespaceInfo.userNamespace, @@ -151,6 +166,8 @@ test("metrics PUT", async (context) => { const server = await createServer({ common: context.common, + app: new Hono(), + routes: [], schema: {}, database, dbNamespace: namespaceInfo.userNamespace, @@ -172,6 +189,8 @@ test("missing route", async (context) => { const server = await createServer({ common: context.common, + app: new Hono(), + routes: [], schema: {}, database, dbNamespace: namespaceInfo.userNamespace, @@ -185,6 +204,54 @@ test("missing route", async (context) => { await cleanup(); }); +test("custom api route", async (context) => { + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); + + const server = await createServer({ + common: context.common, + app: new Hono(), + routes: [ + { method: "GET", pathOrHandlers: ["/hi", (c: Context) => c.text("hi")] }, + ], + schema: {}, + database, + dbNamespace: namespaceInfo.userNamespace, + }); + + const response = await server.hono.request("/hi"); + + expect(response.status).toBe(200); + expect(await response.text()).toBe("hi"); + + await server.kill(); + await cleanup(); +}); + +test("custom hono route", async (context) => { + const { database, namespaceInfo, cleanup } = + await setupDatabaseServices(context); + + const app = new Hono().get("/hi", (c) => c.text("hi")); + + const server = await createServer({ + common: context.common, + app, + routes: [], + schema: {}, + database, + dbNamespace: namespaceInfo.userNamespace, + }); + + const response = await server.hono.request("/hi"); + + expect(response.status).toBe(200); + expect(await response.text()).toBe("hi"); + + await server.kill(); + await cleanup(); +}); + // Note that this test doesn't work because the `hono.request` method doesn't actually // create a socket connection, it just calls the request handler function directly. test.skip("kill", async (context) => { @@ -192,6 +259,8 @@ test.skip("kill", async (context) => { const server = await createServer({ common: context.common, + app: new Hono(), + routes: [], schema: {}, database, dbNamespace: namespaceInfo.userNamespace, diff --git a/packages/core/src/server/service.ts b/packages/core/src/server/service.ts index 9a9af1dcb..4e17264bf 100644 --- a/packages/core/src/server/service.ts +++ b/packages/core/src/server/service.ts @@ -29,8 +29,8 @@ export async function createServer({ database, dbNamespace, }: { - app?: Hono; - routes?: PonderRoutes; + app: Hono; + routes: PonderRoutes; common: Common; schema: Schema; database: DatabaseService; @@ -144,7 +144,11 @@ export async function createServer({ }) .use(contextMiddleware); - if (userApp !== undefined && userRoutes !== undefined) { + if (userRoutes.length === 0 && userApp.routes.length === 0) { + // apply graphql middleware if no custom api exists + hono.use("/graphql", graphql()); + hono.use("/", graphql()); + } else { // apply user routes to hono instance, registering a custom error handler applyHonoRoutes(hono, userRoutes, { db, tables }).onError((error, c) => onError(error, c, common), @@ -157,10 +161,8 @@ export async function createServer({ .filter((maybePathOrHandler) => typeof maybePathOrHandler === "string") .join(", ")}]`, }); - } else { - // apply graphql middleware if no custom api exists - hono.use("/graphql", graphql()); - hono.use("/", graphql()); + + hono.route("/", userApp); } // Create nodejs server From 7b3466b2f8b86fa7f9e95c4fd76a0f2ce4c945ec Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 17 Jul 2024 14:19:06 -0400 Subject: [PATCH 116/122] new example for feature api functions --- .changeset/config.json | 1 + examples/feature-api-functions/.env.example | 5 + examples/feature-api-functions/.eslintrc.json | 3 + examples/feature-api-functions/.gitignore | 18 + .../abis/PrimitiveManagerAbi.ts | 797 ++++++++++++++++++ examples/feature-api-functions/package.json | 27 + .../feature-api-functions/ponder-env.d.ts | 28 + .../feature-api-functions/ponder.config.ts | 21 + .../feature-api-functions/ponder.schema.ts | 9 + .../feature-api-functions/src/api/index.ts | 21 + examples/feature-api-functions/src/index.ts | 13 + examples/feature-api-functions/tsconfig.json | 26 + examples/feature-blocks/package.json | 1 + examples/feature-call-traces/package.json | 1 + examples/feature-factory/package.json | 1 + examples/feature-filter/package.json | 1 + examples/feature-multichain/package.json | 1 + examples/feature-proxy/package.json | 1 + examples/feature-read-contract/package.json | 1 + examples/project-friendtech/package.json | 1 + .../project-uniswap-v3-flash/package.json | 1 + examples/reference-erc1155/package.json | 1 + examples/reference-erc20/package.json | 2 +- examples/reference-erc20/src/api/index.ts | 19 - examples/reference-erc4626/package.json | 1 + examples/reference-erc721/package.json | 1 + examples/with-foundry/ponder/package.json | 1 + examples/with-nextjs/ponder/package.json | 1 + examples/with-trpc/ponder/package.json | 1 + packages/create-ponder/src/index.ts | 5 + pnpm-lock.yaml | 72 +- 31 files changed, 1061 insertions(+), 21 deletions(-) create mode 100644 examples/feature-api-functions/.env.example create mode 100644 examples/feature-api-functions/.eslintrc.json create mode 100644 examples/feature-api-functions/.gitignore create mode 100644 examples/feature-api-functions/abis/PrimitiveManagerAbi.ts create mode 100644 examples/feature-api-functions/package.json create mode 100644 examples/feature-api-functions/ponder-env.d.ts create mode 100644 examples/feature-api-functions/ponder.config.ts create mode 100644 examples/feature-api-functions/ponder.schema.ts create mode 100644 examples/feature-api-functions/src/api/index.ts create mode 100644 examples/feature-api-functions/src/index.ts create mode 100644 examples/feature-api-functions/tsconfig.json delete mode 100644 examples/reference-erc20/src/api/index.ts diff --git a/.changeset/config.json b/.changeset/config.json index 7cf94a65f..b6606880a 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -8,6 +8,7 @@ "ponder-examples-feature-blocks", "ponder-examples-feature-factory", "ponder-examples-feature-filter", + "ponder-examples-feature-api-functions", "ponder-examples-feature-multichain", "ponder-examples-feature-call-traces", "ponder-examples-feature-proxy", diff --git a/examples/feature-api-functions/.env.example b/examples/feature-api-functions/.env.example new file mode 100644 index 000000000..f7745c21c --- /dev/null +++ b/examples/feature-api-functions/.env.example @@ -0,0 +1,5 @@ +# Mainnet RPC URL used for fetching blockchain data. Alchemy is recommended. +PONDER_RPC_URL_1=https://eth-mainnet.g.alchemy.com/v2/... + +# (Optional) Postgres database URL. If not provided, SQLite will be used. +DATABASE_URL= \ No newline at end of file diff --git a/examples/feature-api-functions/.eslintrc.json b/examples/feature-api-functions/.eslintrc.json new file mode 100644 index 000000000..359e2bbfa --- /dev/null +++ b/examples/feature-api-functions/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "ponder" +} diff --git a/examples/feature-api-functions/.gitignore b/examples/feature-api-functions/.gitignore new file mode 100644 index 000000000..f0c7e1177 --- /dev/null +++ b/examples/feature-api-functions/.gitignore @@ -0,0 +1,18 @@ +# Dependencies +/node_modules + +# Debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# Misc +.DS_Store + +# Env files +.env*.local + +# Ponder +/generated/ +/.ponder/ diff --git a/examples/feature-api-functions/abis/PrimitiveManagerAbi.ts b/examples/feature-api-functions/abis/PrimitiveManagerAbi.ts new file mode 100644 index 000000000..47c01560f --- /dev/null +++ b/examples/feature-api-functions/abis/PrimitiveManagerAbi.ts @@ -0,0 +1,797 @@ +export const PrimitiveManagerAbi = [ + { + inputs: [ + { internalType: "address", name: "factory_", type: "address" }, + { internalType: "address", name: "WETH9_", type: "address" }, + { internalType: "address", name: "positionDescriptor_", type: "address" }, + ], + stateMutability: "nonpayable", + type: "constructor", + }, + { + inputs: [ + { internalType: "uint256", name: "balance", type: "uint256" }, + { internalType: "uint256", name: "requiredAmount", type: "uint256" }, + ], + name: "BalanceTooLowError", + type: "error", + }, + { inputs: [], name: "DeadlineReachedError", type: "error" }, + { inputs: [], name: "EngineNotDeployedError", type: "error" }, + { inputs: [], name: "InvalidSigError", type: "error" }, + { inputs: [], name: "LockedError", type: "error" }, + { inputs: [], name: "MinLiquidityOutError", type: "error" }, + { inputs: [], name: "MinRemoveOutError", type: "error" }, + { inputs: [], name: "NotEngineError", type: "error" }, + { inputs: [], name: "OnlyWETHError", type: "error" }, + { inputs: [], name: "SigExpiredError", type: "error" }, + { inputs: [], name: "TransferError", type: "error" }, + { inputs: [], name: "WrongConstructorParametersError", type: "error" }, + { inputs: [], name: "ZeroDelError", type: "error" }, + { inputs: [], name: "ZeroLiquidityError", type: "error" }, + { + anonymous: false, + inputs: [ + { + indexed: false, + internalType: "address", + name: "payer", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "recipient", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "engine", + type: "address", + }, + { + indexed: true, + internalType: "bytes32", + name: "poolId", + type: "bytes32", + }, + { + indexed: false, + internalType: "uint256", + name: "delLiquidity", + type: "uint256", + }, + { + indexed: false, + internalType: "uint256", + name: "delRisky", + type: "uint256", + }, + { + indexed: false, + internalType: "uint256", + name: "delStable", + type: "uint256", + }, + { + indexed: false, + internalType: "bool", + name: "fromMargin", + type: "bool", + }, + ], + name: "Allocate", + type: "event", + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: "address", + name: "account", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "operator", + type: "address", + }, + { indexed: false, internalType: "bool", name: "approved", type: "bool" }, + ], + name: "ApprovalForAll", + type: "event", + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: "address", + name: "payer", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "engine", + type: "address", + }, + { + indexed: true, + internalType: "bytes32", + name: "poolId", + type: "bytes32", + }, + { + indexed: false, + internalType: "uint128", + name: "strike", + type: "uint128", + }, + { indexed: false, internalType: "uint32", name: "sigma", type: "uint32" }, + { + indexed: false, + internalType: "uint32", + name: "maturity", + type: "uint32", + }, + { indexed: false, internalType: "uint32", name: "gamma", type: "uint32" }, + { + indexed: false, + internalType: "uint256", + name: "delLiquidity", + type: "uint256", + }, + ], + name: "Create", + type: "event", + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: "address", + name: "payer", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "recipient", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "engine", + type: "address", + }, + { + indexed: false, + internalType: "address", + name: "risky", + type: "address", + }, + { + indexed: false, + internalType: "address", + name: "stable", + type: "address", + }, + { + indexed: false, + internalType: "uint256", + name: "delRisky", + type: "uint256", + }, + { + indexed: false, + internalType: "uint256", + name: "delStable", + type: "uint256", + }, + ], + name: "Deposit", + type: "event", + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: "address", + name: "payer", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "engine", + type: "address", + }, + { + indexed: true, + internalType: "bytes32", + name: "poolId", + type: "bytes32", + }, + { + indexed: false, + internalType: "uint256", + name: "delLiquidity", + type: "uint256", + }, + { + indexed: false, + internalType: "uint256", + name: "delRisky", + type: "uint256", + }, + { + indexed: false, + internalType: "uint256", + name: "delStable", + type: "uint256", + }, + ], + name: "Remove", + type: "event", + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: "address", + name: "payer", + type: "address", + }, + { + indexed: false, + internalType: "address", + name: "recipient", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "engine", + type: "address", + }, + { + indexed: true, + internalType: "bytes32", + name: "poolId", + type: "bytes32", + }, + { + indexed: false, + internalType: "bool", + name: "riskyForStable", + type: "bool", + }, + { + indexed: false, + internalType: "uint256", + name: "deltaIn", + type: "uint256", + }, + { + indexed: false, + internalType: "uint256", + name: "deltaOut", + type: "uint256", + }, + { + indexed: false, + internalType: "bool", + name: "fromMargin", + type: "bool", + }, + { indexed: false, internalType: "bool", name: "toMargin", type: "bool" }, + ], + name: "Swap", + type: "event", + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: "address", + name: "operator", + type: "address", + }, + { indexed: true, internalType: "address", name: "from", type: "address" }, + { indexed: true, internalType: "address", name: "to", type: "address" }, + { + indexed: false, + internalType: "uint256[]", + name: "ids", + type: "uint256[]", + }, + { + indexed: false, + internalType: "uint256[]", + name: "values", + type: "uint256[]", + }, + ], + name: "TransferBatch", + type: "event", + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: "address", + name: "operator", + type: "address", + }, + { indexed: true, internalType: "address", name: "from", type: "address" }, + { indexed: true, internalType: "address", name: "to", type: "address" }, + { indexed: false, internalType: "uint256", name: "id", type: "uint256" }, + { + indexed: false, + internalType: "uint256", + name: "value", + type: "uint256", + }, + ], + name: "TransferSingle", + type: "event", + }, + { + anonymous: false, + inputs: [ + { indexed: false, internalType: "string", name: "value", type: "string" }, + { indexed: true, internalType: "uint256", name: "id", type: "uint256" }, + ], + name: "URI", + type: "event", + }, + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: "address", + name: "payer", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "recipient", + type: "address", + }, + { + indexed: true, + internalType: "address", + name: "engine", + type: "address", + }, + { + indexed: false, + internalType: "address", + name: "risky", + type: "address", + }, + { + indexed: false, + internalType: "address", + name: "stable", + type: "address", + }, + { + indexed: false, + internalType: "uint256", + name: "delRisky", + type: "uint256", + }, + { + indexed: false, + internalType: "uint256", + name: "delStable", + type: "uint256", + }, + ], + name: "Withdraw", + type: "event", + }, + { + inputs: [], + name: "DOMAIN_SEPARATOR", + outputs: [{ internalType: "bytes32", name: "", type: "bytes32" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [], + name: "WETH9", + outputs: [{ internalType: "address", name: "", type: "address" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "recipient", type: "address" }, + { internalType: "bytes32", name: "poolId", type: "bytes32" }, + { internalType: "address", name: "risky", type: "address" }, + { internalType: "address", name: "stable", type: "address" }, + { internalType: "uint256", name: "delRisky", type: "uint256" }, + { internalType: "uint256", name: "delStable", type: "uint256" }, + { internalType: "bool", name: "fromMargin", type: "bool" }, + { internalType: "uint256", name: "minLiquidityOut", type: "uint256" }, + ], + name: "allocate", + outputs: [ + { internalType: "uint256", name: "delLiquidity", type: "uint256" }, + ], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "uint256", name: "delRisky", type: "uint256" }, + { internalType: "uint256", name: "delStable", type: "uint256" }, + { internalType: "bytes", name: "data", type: "bytes" }, + ], + name: "allocateCallback", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "account", type: "address" }, + { internalType: "uint256", name: "id", type: "uint256" }, + ], + name: "balanceOf", + outputs: [{ internalType: "uint256", name: "", type: "uint256" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { internalType: "address[]", name: "accounts", type: "address[]" }, + { internalType: "uint256[]", name: "ids", type: "uint256[]" }, + ], + name: "balanceOfBatch", + outputs: [{ internalType: "uint256[]", name: "", type: "uint256[]" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "risky", type: "address" }, + { internalType: "address", name: "stable", type: "address" }, + { internalType: "uint128", name: "strike", type: "uint128" }, + { internalType: "uint32", name: "sigma", type: "uint32" }, + { internalType: "uint32", name: "maturity", type: "uint32" }, + { internalType: "uint32", name: "gamma", type: "uint32" }, + { internalType: "uint256", name: "riskyPerLp", type: "uint256" }, + { internalType: "uint256", name: "delLiquidity", type: "uint256" }, + ], + name: "create", + outputs: [ + { internalType: "bytes32", name: "poolId", type: "bytes32" }, + { internalType: "uint256", name: "delRisky", type: "uint256" }, + { internalType: "uint256", name: "delStable", type: "uint256" }, + ], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "uint256", name: "delRisky", type: "uint256" }, + { internalType: "uint256", name: "delStable", type: "uint256" }, + { internalType: "bytes", name: "data", type: "bytes" }, + ], + name: "createCallback", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "recipient", type: "address" }, + { internalType: "address", name: "risky", type: "address" }, + { internalType: "address", name: "stable", type: "address" }, + { internalType: "uint256", name: "delRisky", type: "uint256" }, + { internalType: "uint256", name: "delStable", type: "uint256" }, + ], + name: "deposit", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "uint256", name: "delRisky", type: "uint256" }, + { internalType: "uint256", name: "delStable", type: "uint256" }, + { internalType: "bytes", name: "data", type: "bytes" }, + ], + name: "depositCallback", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [], + name: "factory", + outputs: [{ internalType: "address", name: "", type: "address" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "account", type: "address" }, + { internalType: "address", name: "operator", type: "address" }, + ], + name: "isApprovedForAll", + outputs: [{ internalType: "bool", name: "", type: "bool" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "", type: "address" }, + { internalType: "address", name: "", type: "address" }, + ], + name: "margins", + outputs: [ + { internalType: "uint128", name: "balanceRisky", type: "uint128" }, + { internalType: "uint128", name: "balanceStable", type: "uint128" }, + ], + stateMutability: "view", + type: "function", + }, + { + inputs: [{ internalType: "bytes[]", name: "data", type: "bytes[]" }], + name: "multicall", + outputs: [{ internalType: "bytes[]", name: "results", type: "bytes[]" }], + stateMutability: "payable", + type: "function", + }, + { + inputs: [{ internalType: "address", name: "", type: "address" }], + name: "nonces", + outputs: [{ internalType: "uint256", name: "", type: "uint256" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "owner", type: "address" }, + { internalType: "address", name: "operator", type: "address" }, + { internalType: "bool", name: "approved", type: "bool" }, + { internalType: "uint256", name: "deadline", type: "uint256" }, + { internalType: "uint8", name: "v", type: "uint8" }, + { internalType: "bytes32", name: "r", type: "bytes32" }, + { internalType: "bytes32", name: "s", type: "bytes32" }, + ], + name: "permit", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [], + name: "positionDescriptor", + outputs: [{ internalType: "address", name: "", type: "address" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [], + name: "refundETH", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "engine", type: "address" }, + { internalType: "bytes32", name: "poolId", type: "bytes32" }, + { internalType: "uint256", name: "delLiquidity", type: "uint256" }, + { internalType: "uint256", name: "minRiskyOut", type: "uint256" }, + { internalType: "uint256", name: "minStableOut", type: "uint256" }, + ], + name: "remove", + outputs: [ + { internalType: "uint256", name: "delRisky", type: "uint256" }, + { internalType: "uint256", name: "delStable", type: "uint256" }, + ], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "from", type: "address" }, + { internalType: "address", name: "to", type: "address" }, + { internalType: "uint256[]", name: "ids", type: "uint256[]" }, + { internalType: "uint256[]", name: "amounts", type: "uint256[]" }, + { internalType: "bytes", name: "data", type: "bytes" }, + ], + name: "safeBatchTransferFrom", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "from", type: "address" }, + { internalType: "address", name: "to", type: "address" }, + { internalType: "uint256", name: "id", type: "uint256" }, + { internalType: "uint256", name: "amount", type: "uint256" }, + { internalType: "bytes", name: "data", type: "bytes" }, + ], + name: "safeTransferFrom", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "token", type: "address" }, + { internalType: "uint256", name: "value", type: "uint256" }, + { internalType: "uint256", name: "deadline", type: "uint256" }, + { internalType: "uint8", name: "v", type: "uint8" }, + { internalType: "bytes32", name: "r", type: "bytes32" }, + { internalType: "bytes32", name: "s", type: "bytes32" }, + ], + name: "selfPermit", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "token", type: "address" }, + { internalType: "uint256", name: "nonce", type: "uint256" }, + { internalType: "uint256", name: "expiry", type: "uint256" }, + { internalType: "uint8", name: "v", type: "uint8" }, + { internalType: "bytes32", name: "r", type: "bytes32" }, + { internalType: "bytes32", name: "s", type: "bytes32" }, + ], + name: "selfPermitAllowed", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "token", type: "address" }, + { internalType: "uint256", name: "nonce", type: "uint256" }, + { internalType: "uint256", name: "expiry", type: "uint256" }, + { internalType: "uint8", name: "v", type: "uint8" }, + { internalType: "bytes32", name: "r", type: "bytes32" }, + { internalType: "bytes32", name: "s", type: "bytes32" }, + ], + name: "selfPermitAllowedIfNecessary", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "token", type: "address" }, + { internalType: "uint256", name: "value", type: "uint256" }, + { internalType: "uint256", name: "deadline", type: "uint256" }, + { internalType: "uint8", name: "v", type: "uint8" }, + { internalType: "bytes32", name: "r", type: "bytes32" }, + { internalType: "bytes32", name: "s", type: "bytes32" }, + ], + name: "selfPermitIfNecessary", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "operator", type: "address" }, + { internalType: "bool", name: "approved", type: "bool" }, + ], + name: "setApprovalForAll", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [{ internalType: "bytes4", name: "interfaceId", type: "bytes4" }], + name: "supportsInterface", + outputs: [{ internalType: "bool", name: "", type: "bool" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { + components: [ + { internalType: "address", name: "recipient", type: "address" }, + { internalType: "address", name: "risky", type: "address" }, + { internalType: "address", name: "stable", type: "address" }, + { internalType: "bytes32", name: "poolId", type: "bytes32" }, + { internalType: "bool", name: "riskyForStable", type: "bool" }, + { internalType: "uint256", name: "deltaIn", type: "uint256" }, + { internalType: "uint256", name: "deltaOut", type: "uint256" }, + { internalType: "bool", name: "fromMargin", type: "bool" }, + { internalType: "bool", name: "toMargin", type: "bool" }, + { internalType: "uint256", name: "deadline", type: "uint256" }, + ], + internalType: "struct ISwapManager.SwapParams", + name: "params", + type: "tuple", + }, + ], + name: "swap", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "uint256", name: "delRisky", type: "uint256" }, + { internalType: "uint256", name: "delStable", type: "uint256" }, + { internalType: "bytes", name: "data", type: "bytes" }, + ], + name: "swapCallback", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "token", type: "address" }, + { internalType: "uint256", name: "amountMin", type: "uint256" }, + { internalType: "address", name: "recipient", type: "address" }, + ], + name: "sweepToken", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { + inputs: [ + { internalType: "uint256", name: "amountMin", type: "uint256" }, + { internalType: "address", name: "recipient", type: "address" }, + ], + name: "unwrap", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { + inputs: [{ internalType: "uint256", name: "tokenId", type: "uint256" }], + name: "uri", + outputs: [{ internalType: "string", name: "", type: "string" }], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { internalType: "address", name: "recipient", type: "address" }, + { internalType: "address", name: "engine", type: "address" }, + { internalType: "uint256", name: "delRisky", type: "uint256" }, + { internalType: "uint256", name: "delStable", type: "uint256" }, + ], + name: "withdraw", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, + { + inputs: [{ internalType: "uint256", name: "value", type: "uint256" }], + name: "wrap", + outputs: [], + stateMutability: "payable", + type: "function", + }, + { stateMutability: "payable", type: "receive" }, +] as const; diff --git a/examples/feature-api-functions/package.json b/examples/feature-api-functions/package.json new file mode 100644 index 000000000..9578a0a34 --- /dev/null +++ b/examples/feature-api-functions/package.json @@ -0,0 +1,27 @@ +{ + "name": "ponder-examples-feature-api-functions", + "private": true, + "type": "module", + "scripts": { + "dev": "ponder dev", + "start": "ponder start", + "codegen": "ponder codegen", + "serve": "ponder serve", + "lint": "eslint .", + "typecheck": "tsc" + }, + "dependencies": { + "@ponder/core": "workspace:*", + "hono": "^4.5.0", + "viem": "^1.19.9" + }, + "devDependencies": { + "@types/node": "^20.10.0", + "eslint": "^8.54.0", + "eslint-config-ponder": "workspace:*", + "typescript": "^5.3.2" + }, + "engines": { + "node": ">=18.14" + } +} diff --git a/examples/feature-api-functions/ponder-env.d.ts b/examples/feature-api-functions/ponder-env.d.ts new file mode 100644 index 000000000..03126bf92 --- /dev/null +++ b/examples/feature-api-functions/ponder-env.d.ts @@ -0,0 +1,28 @@ +// This file enables type checking and editor autocomplete for this Ponder project. +// After upgrading, you may find that changes have been made to this file. +// If this happens, please commit the changes. Do not manually edit this file. +// See https://ponder.sh/docs/getting-started/installation#typescript for more information. + +declare module "@/generated" { + import type { Virtual } from "@ponder/core"; + + type config = typeof import("./ponder.config.ts").default; + type schema = typeof import("./ponder.schema.ts").default; + + export const ponder: Virtual.Registry; + + export type EventNames = Virtual.EventNames; + export type Event = Virtual.Event< + config, + name + >; + export type Context = Virtual.Context< + config, + schema, + name + >; + export type ApiContext = Virtual.Drizzle; + export type IndexingFunctionArgs = + Virtual.IndexingFunctionArgs; + export type Schema = Virtual.Schema; +} diff --git a/examples/feature-api-functions/ponder.config.ts b/examples/feature-api-functions/ponder.config.ts new file mode 100644 index 000000000..8494f8856 --- /dev/null +++ b/examples/feature-api-functions/ponder.config.ts @@ -0,0 +1,21 @@ +import { createConfig } from "@ponder/core"; +import { http } from "viem"; + +import { PrimitiveManagerAbi } from "./abis/PrimitiveManagerAbi"; + +export default createConfig({ + networks: { + mainnet: { + chainId: 1, + transport: http(process.env.PONDER_RPC_URL_1), + }, + }, + contracts: { + PrimitiveManager: { + network: "mainnet", + abi: PrimitiveManagerAbi, + address: "0x54522dA62a15225C95b01bD61fF58b866C50471f", + startBlock: 14438081, + }, + }, +}); diff --git a/examples/feature-api-functions/ponder.schema.ts b/examples/feature-api-functions/ponder.schema.ts new file mode 100644 index 000000000..6a3627629 --- /dev/null +++ b/examples/feature-api-functions/ponder.schema.ts @@ -0,0 +1,9 @@ +import { createSchema } from "@ponder/core"; + +export default createSchema((p) => ({ + SwapEvent: p.createTable({ + id: p.string(), + recipient: p.hex(), + payer: p.hex(), + }), +})); diff --git a/examples/feature-api-functions/src/api/index.ts b/examples/feature-api-functions/src/api/index.ts new file mode 100644 index 000000000..bc24a9557 --- /dev/null +++ b/examples/feature-api-functions/src/api/index.ts @@ -0,0 +1,21 @@ +import { ponder } from "@/generated"; +import { count, eq, or } from "@ponder/core"; +import { getAddress } from "viem"; + +ponder.get("/count", async (c) => { + const result = await c.db.select({ count: count() }).from(c.tables.SwapEvent); + + return c.text(String(result[0]?.count ?? 0)); +}); + +ponder.get("/user-count/:address", async (c) => { + const account = getAddress(c.req.param("address")); + const { SwapEvent } = c.tables; + + const result = await c.db + .select({ count: count() }) + .from(c.tables.SwapEvent) + .where(or(eq(SwapEvent.payer, account), eq(SwapEvent.recipient, account))); + + return c.text(String(result[0]?.count ?? 0)); +}); diff --git a/examples/feature-api-functions/src/index.ts b/examples/feature-api-functions/src/index.ts new file mode 100644 index 000000000..ce46fbbcb --- /dev/null +++ b/examples/feature-api-functions/src/index.ts @@ -0,0 +1,13 @@ +import { ponder } from "@/generated"; + +ponder.on("PrimitiveManager:Swap", async ({ event, context }) => { + const { SwapEvent } = context.db; + + await SwapEvent.create({ + id: event.log.id, + data: { + payer: event.args.payer, + recipient: event.args.recipient, + }, + }); +}); diff --git a/examples/feature-api-functions/tsconfig.json b/examples/feature-api-functions/tsconfig.json new file mode 100644 index 000000000..592b9a939 --- /dev/null +++ b/examples/feature-api-functions/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + // Type checking + "strict": true, + "noUncheckedIndexedAccess": true, + + // Interop constraints + "verbatimModuleSyntax": false, + "esModuleInterop": true, + "isolatedModules": true, + "allowSyntheticDefaultImports": true, + "resolveJsonModule": true, + + // Language and environment + "moduleResolution": "bundler", + "module": "ESNext", + "noEmit": true, + "lib": ["ES2022"], + "target": "ES2022", + + // Skip type checking for node modules + "skipLibCheck": true + }, + "include": ["./**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/examples/feature-blocks/package.json b/examples/feature-blocks/package.json index 30ac311bb..0ee3320c7 100644 --- a/examples/feature-blocks/package.json +++ b/examples/feature-blocks/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/feature-call-traces/package.json b/examples/feature-call-traces/package.json index 8cb804b97..47495ec9a 100644 --- a/examples/feature-call-traces/package.json +++ b/examples/feature-call-traces/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/feature-factory/package.json b/examples/feature-factory/package.json index 910df7367..40a0a0f2d 100644 --- a/examples/feature-factory/package.json +++ b/examples/feature-factory/package.json @@ -13,6 +13,7 @@ "dependencies": { "@ponder/core": "workspace:*", "abitype": "^0.10.2", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/feature-filter/package.json b/examples/feature-filter/package.json index 2f06346e6..7d67fddaf 100644 --- a/examples/feature-filter/package.json +++ b/examples/feature-filter/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/feature-multichain/package.json b/examples/feature-multichain/package.json index 0ca701d42..242ffd29d 100644 --- a/examples/feature-multichain/package.json +++ b/examples/feature-multichain/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/feature-proxy/package.json b/examples/feature-proxy/package.json index 01f8f305f..679e8bf32 100644 --- a/examples/feature-proxy/package.json +++ b/examples/feature-proxy/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/feature-read-contract/package.json b/examples/feature-read-contract/package.json index 6a833a937..6107ee15f 100644 --- a/examples/feature-read-contract/package.json +++ b/examples/feature-read-contract/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/project-friendtech/package.json b/examples/project-friendtech/package.json index 90cb0c488..de1ac3979 100644 --- a/examples/project-friendtech/package.json +++ b/examples/project-friendtech/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/project-uniswap-v3-flash/package.json b/examples/project-uniswap-v3-flash/package.json index 01e060539..c6c8696b7 100644 --- a/examples/project-uniswap-v3-flash/package.json +++ b/examples/project-uniswap-v3-flash/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/reference-erc1155/package.json b/examples/reference-erc1155/package.json index fbb297929..e72e4e9a8 100644 --- a/examples/reference-erc1155/package.json +++ b/examples/reference-erc1155/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/reference-erc20/package.json b/examples/reference-erc20/package.json index 8e893e9f6..dcf42502f 100644 --- a/examples/reference-erc20/package.json +++ b/examples/reference-erc20/package.json @@ -12,7 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", - "hono": "^4.4.6", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/reference-erc20/src/api/index.ts b/examples/reference-erc20/src/api/index.ts deleted file mode 100644 index 017cbdc6a..000000000 --- a/examples/reference-erc20/src/api/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { ponder } from "@/generated"; -import { desc, graphql } from "@ponder/core"; -import { formatEther } from "viem"; - -ponder.use("/graphql", graphql()).get("/big", async (c) => { - const { Account } = c.tables; - - const account = await c.db - .select({ balance: Account.balance }) - .from(Account) - .orderBy(desc(Account.balance)) - .limit(1); - - if (account.length === 0) { - return c.text("Not Found!"); - } else { - return c.text(`Balance: ${formatEther(account[0]!.balance)}`); - } -}); diff --git a/examples/reference-erc4626/package.json b/examples/reference-erc4626/package.json index b504c9b72..9c7a2ffeb 100644 --- a/examples/reference-erc4626/package.json +++ b/examples/reference-erc4626/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/reference-erc721/package.json b/examples/reference-erc721/package.json index 879b2ab64..edd6b2abe 100644 --- a/examples/reference-erc721/package.json +++ b/examples/reference-erc721/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/with-foundry/ponder/package.json b/examples/with-foundry/ponder/package.json index 5811c4c65..1f57f0646 100644 --- a/examples/with-foundry/ponder/package.json +++ b/examples/with-foundry/ponder/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.3" }, "devDependencies": { diff --git a/examples/with-nextjs/ponder/package.json b/examples/with-nextjs/ponder/package.json index 83a129df0..fd47071e8 100644 --- a/examples/with-nextjs/ponder/package.json +++ b/examples/with-nextjs/ponder/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@ponder/core": "workspace:*", + "hono": "^4.5.0", "viem": "^1.19.9" }, "devDependencies": { diff --git a/examples/with-trpc/ponder/package.json b/examples/with-trpc/ponder/package.json index 7c9c27c34..e0f3aa491 100644 --- a/examples/with-trpc/ponder/package.json +++ b/examples/with-trpc/ponder/package.json @@ -13,6 +13,7 @@ "@hono/trpc-server": "^0.3.2", "@ponder/core": "workspace:*", "@trpc/server": "^10.45.2", + "hono": "^4.5.0", "viem": "^1.19.9", "zod": "^3.23.8" }, diff --git a/packages/create-ponder/src/index.ts b/packages/create-ponder/src/index.ts index e51174338..fc03bab3c 100644 --- a/packages/create-ponder/src/index.ts +++ b/packages/create-ponder/src/index.ts @@ -80,6 +80,11 @@ const templates = [ title: "Feature - Custom event filter", description: "A Ponder app using an event filter", }, + { + id: "feature-api-functions", + title: "Feature - Custom api functions", + description: "A Ponder app using a custom api functions", + }, { id: "feature-blocks", title: "Feature - Block filter", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4f05809d8..624a83907 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -141,11 +141,39 @@ importers: specifier: ^5.2.2 version: 5.3.3 + examples/feature-api-functions: + dependencies: + '@ponder/core': + specifier: workspace:* + version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 + viem: + specifier: ^1.19.9 + version: 1.21.4(typescript@5.3.3)(zod@3.23.8) + devDependencies: + '@types/node': + specifier: ^20.10.0 + version: 20.11.24 + eslint: + specifier: ^8.54.0 + version: 8.56.0 + eslint-config-ponder: + specifier: workspace:* + version: link:../../packages/eslint-config-ponder + typescript: + specifier: ^5.3.2 + version: 5.3.3 + examples/feature-blocks: dependencies: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -168,6 +196,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -193,6 +224,9 @@ importers: abitype: specifier: ^0.10.2 version: 0.10.3(typescript@5.3.3) + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -215,6 +249,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -237,6 +274,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -259,6 +299,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -281,6 +324,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -303,6 +349,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -325,6 +374,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -347,6 +399,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -370,7 +425,7 @@ importers: specifier: workspace:* version: link:../../packages/core hono: - specifier: ^4.4.6 + specifier: ^4.5.0 version: 4.5.0 viem: specifier: ^1.19.9 @@ -394,6 +449,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -416,6 +474,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -450,6 +511,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.3 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -529,6 +593,9 @@ importers: '@ponder/core': specifier: workspace:* version: link:../../../packages/core + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) @@ -565,6 +632,9 @@ importers: '@trpc/server': specifier: ^10.45.2 version: 10.45.2 + hono: + specifier: ^4.5.0 + version: 4.5.0 viem: specifier: ^1.19.9 version: 1.21.4(typescript@5.3.3)(zod@3.23.8) From 4822f8e3c8dab58d74e9ebe7465e274e2ae71c51 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 17 Jul 2024 14:48:24 -0400 Subject: [PATCH 117/122] update api functions example --- .../abis/PrimitiveManagerAbi.ts | 797 ------------------ .../feature-api-functions/abis/erc20ABI.ts | 147 ++++ .../feature-api-functions/ponder.config.ts | 12 +- .../feature-api-functions/ponder.schema.ts | 47 +- .../feature-api-functions/src/api/index.ts | 44 +- examples/feature-api-functions/src/index.ts | 67 +- 6 files changed, 294 insertions(+), 820 deletions(-) delete mode 100644 examples/feature-api-functions/abis/PrimitiveManagerAbi.ts create mode 100644 examples/feature-api-functions/abis/erc20ABI.ts diff --git a/examples/feature-api-functions/abis/PrimitiveManagerAbi.ts b/examples/feature-api-functions/abis/PrimitiveManagerAbi.ts deleted file mode 100644 index 47c01560f..000000000 --- a/examples/feature-api-functions/abis/PrimitiveManagerAbi.ts +++ /dev/null @@ -1,797 +0,0 @@ -export const PrimitiveManagerAbi = [ - { - inputs: [ - { internalType: "address", name: "factory_", type: "address" }, - { internalType: "address", name: "WETH9_", type: "address" }, - { internalType: "address", name: "positionDescriptor_", type: "address" }, - ], - stateMutability: "nonpayable", - type: "constructor", - }, - { - inputs: [ - { internalType: "uint256", name: "balance", type: "uint256" }, - { internalType: "uint256", name: "requiredAmount", type: "uint256" }, - ], - name: "BalanceTooLowError", - type: "error", - }, - { inputs: [], name: "DeadlineReachedError", type: "error" }, - { inputs: [], name: "EngineNotDeployedError", type: "error" }, - { inputs: [], name: "InvalidSigError", type: "error" }, - { inputs: [], name: "LockedError", type: "error" }, - { inputs: [], name: "MinLiquidityOutError", type: "error" }, - { inputs: [], name: "MinRemoveOutError", type: "error" }, - { inputs: [], name: "NotEngineError", type: "error" }, - { inputs: [], name: "OnlyWETHError", type: "error" }, - { inputs: [], name: "SigExpiredError", type: "error" }, - { inputs: [], name: "TransferError", type: "error" }, - { inputs: [], name: "WrongConstructorParametersError", type: "error" }, - { inputs: [], name: "ZeroDelError", type: "error" }, - { inputs: [], name: "ZeroLiquidityError", type: "error" }, - { - anonymous: false, - inputs: [ - { - indexed: false, - internalType: "address", - name: "payer", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "recipient", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "engine", - type: "address", - }, - { - indexed: true, - internalType: "bytes32", - name: "poolId", - type: "bytes32", - }, - { - indexed: false, - internalType: "uint256", - name: "delLiquidity", - type: "uint256", - }, - { - indexed: false, - internalType: "uint256", - name: "delRisky", - type: "uint256", - }, - { - indexed: false, - internalType: "uint256", - name: "delStable", - type: "uint256", - }, - { - indexed: false, - internalType: "bool", - name: "fromMargin", - type: "bool", - }, - ], - name: "Allocate", - type: "event", - }, - { - anonymous: false, - inputs: [ - { - indexed: true, - internalType: "address", - name: "account", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "operator", - type: "address", - }, - { indexed: false, internalType: "bool", name: "approved", type: "bool" }, - ], - name: "ApprovalForAll", - type: "event", - }, - { - anonymous: false, - inputs: [ - { - indexed: true, - internalType: "address", - name: "payer", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "engine", - type: "address", - }, - { - indexed: true, - internalType: "bytes32", - name: "poolId", - type: "bytes32", - }, - { - indexed: false, - internalType: "uint128", - name: "strike", - type: "uint128", - }, - { indexed: false, internalType: "uint32", name: "sigma", type: "uint32" }, - { - indexed: false, - internalType: "uint32", - name: "maturity", - type: "uint32", - }, - { indexed: false, internalType: "uint32", name: "gamma", type: "uint32" }, - { - indexed: false, - internalType: "uint256", - name: "delLiquidity", - type: "uint256", - }, - ], - name: "Create", - type: "event", - }, - { - anonymous: false, - inputs: [ - { - indexed: true, - internalType: "address", - name: "payer", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "recipient", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "engine", - type: "address", - }, - { - indexed: false, - internalType: "address", - name: "risky", - type: "address", - }, - { - indexed: false, - internalType: "address", - name: "stable", - type: "address", - }, - { - indexed: false, - internalType: "uint256", - name: "delRisky", - type: "uint256", - }, - { - indexed: false, - internalType: "uint256", - name: "delStable", - type: "uint256", - }, - ], - name: "Deposit", - type: "event", - }, - { - anonymous: false, - inputs: [ - { - indexed: true, - internalType: "address", - name: "payer", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "engine", - type: "address", - }, - { - indexed: true, - internalType: "bytes32", - name: "poolId", - type: "bytes32", - }, - { - indexed: false, - internalType: "uint256", - name: "delLiquidity", - type: "uint256", - }, - { - indexed: false, - internalType: "uint256", - name: "delRisky", - type: "uint256", - }, - { - indexed: false, - internalType: "uint256", - name: "delStable", - type: "uint256", - }, - ], - name: "Remove", - type: "event", - }, - { - anonymous: false, - inputs: [ - { - indexed: true, - internalType: "address", - name: "payer", - type: "address", - }, - { - indexed: false, - internalType: "address", - name: "recipient", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "engine", - type: "address", - }, - { - indexed: true, - internalType: "bytes32", - name: "poolId", - type: "bytes32", - }, - { - indexed: false, - internalType: "bool", - name: "riskyForStable", - type: "bool", - }, - { - indexed: false, - internalType: "uint256", - name: "deltaIn", - type: "uint256", - }, - { - indexed: false, - internalType: "uint256", - name: "deltaOut", - type: "uint256", - }, - { - indexed: false, - internalType: "bool", - name: "fromMargin", - type: "bool", - }, - { indexed: false, internalType: "bool", name: "toMargin", type: "bool" }, - ], - name: "Swap", - type: "event", - }, - { - anonymous: false, - inputs: [ - { - indexed: true, - internalType: "address", - name: "operator", - type: "address", - }, - { indexed: true, internalType: "address", name: "from", type: "address" }, - { indexed: true, internalType: "address", name: "to", type: "address" }, - { - indexed: false, - internalType: "uint256[]", - name: "ids", - type: "uint256[]", - }, - { - indexed: false, - internalType: "uint256[]", - name: "values", - type: "uint256[]", - }, - ], - name: "TransferBatch", - type: "event", - }, - { - anonymous: false, - inputs: [ - { - indexed: true, - internalType: "address", - name: "operator", - type: "address", - }, - { indexed: true, internalType: "address", name: "from", type: "address" }, - { indexed: true, internalType: "address", name: "to", type: "address" }, - { indexed: false, internalType: "uint256", name: "id", type: "uint256" }, - { - indexed: false, - internalType: "uint256", - name: "value", - type: "uint256", - }, - ], - name: "TransferSingle", - type: "event", - }, - { - anonymous: false, - inputs: [ - { indexed: false, internalType: "string", name: "value", type: "string" }, - { indexed: true, internalType: "uint256", name: "id", type: "uint256" }, - ], - name: "URI", - type: "event", - }, - { - anonymous: false, - inputs: [ - { - indexed: true, - internalType: "address", - name: "payer", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "recipient", - type: "address", - }, - { - indexed: true, - internalType: "address", - name: "engine", - type: "address", - }, - { - indexed: false, - internalType: "address", - name: "risky", - type: "address", - }, - { - indexed: false, - internalType: "address", - name: "stable", - type: "address", - }, - { - indexed: false, - internalType: "uint256", - name: "delRisky", - type: "uint256", - }, - { - indexed: false, - internalType: "uint256", - name: "delStable", - type: "uint256", - }, - ], - name: "Withdraw", - type: "event", - }, - { - inputs: [], - name: "DOMAIN_SEPARATOR", - outputs: [{ internalType: "bytes32", name: "", type: "bytes32" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [], - name: "WETH9", - outputs: [{ internalType: "address", name: "", type: "address" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "recipient", type: "address" }, - { internalType: "bytes32", name: "poolId", type: "bytes32" }, - { internalType: "address", name: "risky", type: "address" }, - { internalType: "address", name: "stable", type: "address" }, - { internalType: "uint256", name: "delRisky", type: "uint256" }, - { internalType: "uint256", name: "delStable", type: "uint256" }, - { internalType: "bool", name: "fromMargin", type: "bool" }, - { internalType: "uint256", name: "minLiquidityOut", type: "uint256" }, - ], - name: "allocate", - outputs: [ - { internalType: "uint256", name: "delLiquidity", type: "uint256" }, - ], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "uint256", name: "delRisky", type: "uint256" }, - { internalType: "uint256", name: "delStable", type: "uint256" }, - { internalType: "bytes", name: "data", type: "bytes" }, - ], - name: "allocateCallback", - outputs: [], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "account", type: "address" }, - { internalType: "uint256", name: "id", type: "uint256" }, - ], - name: "balanceOf", - outputs: [{ internalType: "uint256", name: "", type: "uint256" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [ - { internalType: "address[]", name: "accounts", type: "address[]" }, - { internalType: "uint256[]", name: "ids", type: "uint256[]" }, - ], - name: "balanceOfBatch", - outputs: [{ internalType: "uint256[]", name: "", type: "uint256[]" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "risky", type: "address" }, - { internalType: "address", name: "stable", type: "address" }, - { internalType: "uint128", name: "strike", type: "uint128" }, - { internalType: "uint32", name: "sigma", type: "uint32" }, - { internalType: "uint32", name: "maturity", type: "uint32" }, - { internalType: "uint32", name: "gamma", type: "uint32" }, - { internalType: "uint256", name: "riskyPerLp", type: "uint256" }, - { internalType: "uint256", name: "delLiquidity", type: "uint256" }, - ], - name: "create", - outputs: [ - { internalType: "bytes32", name: "poolId", type: "bytes32" }, - { internalType: "uint256", name: "delRisky", type: "uint256" }, - { internalType: "uint256", name: "delStable", type: "uint256" }, - ], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "uint256", name: "delRisky", type: "uint256" }, - { internalType: "uint256", name: "delStable", type: "uint256" }, - { internalType: "bytes", name: "data", type: "bytes" }, - ], - name: "createCallback", - outputs: [], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "recipient", type: "address" }, - { internalType: "address", name: "risky", type: "address" }, - { internalType: "address", name: "stable", type: "address" }, - { internalType: "uint256", name: "delRisky", type: "uint256" }, - { internalType: "uint256", name: "delStable", type: "uint256" }, - ], - name: "deposit", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "uint256", name: "delRisky", type: "uint256" }, - { internalType: "uint256", name: "delStable", type: "uint256" }, - { internalType: "bytes", name: "data", type: "bytes" }, - ], - name: "depositCallback", - outputs: [], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [], - name: "factory", - outputs: [{ internalType: "address", name: "", type: "address" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "account", type: "address" }, - { internalType: "address", name: "operator", type: "address" }, - ], - name: "isApprovedForAll", - outputs: [{ internalType: "bool", name: "", type: "bool" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "", type: "address" }, - { internalType: "address", name: "", type: "address" }, - ], - name: "margins", - outputs: [ - { internalType: "uint128", name: "balanceRisky", type: "uint128" }, - { internalType: "uint128", name: "balanceStable", type: "uint128" }, - ], - stateMutability: "view", - type: "function", - }, - { - inputs: [{ internalType: "bytes[]", name: "data", type: "bytes[]" }], - name: "multicall", - outputs: [{ internalType: "bytes[]", name: "results", type: "bytes[]" }], - stateMutability: "payable", - type: "function", - }, - { - inputs: [{ internalType: "address", name: "", type: "address" }], - name: "nonces", - outputs: [{ internalType: "uint256", name: "", type: "uint256" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "owner", type: "address" }, - { internalType: "address", name: "operator", type: "address" }, - { internalType: "bool", name: "approved", type: "bool" }, - { internalType: "uint256", name: "deadline", type: "uint256" }, - { internalType: "uint8", name: "v", type: "uint8" }, - { internalType: "bytes32", name: "r", type: "bytes32" }, - { internalType: "bytes32", name: "s", type: "bytes32" }, - ], - name: "permit", - outputs: [], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [], - name: "positionDescriptor", - outputs: [{ internalType: "address", name: "", type: "address" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [], - name: "refundETH", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "engine", type: "address" }, - { internalType: "bytes32", name: "poolId", type: "bytes32" }, - { internalType: "uint256", name: "delLiquidity", type: "uint256" }, - { internalType: "uint256", name: "minRiskyOut", type: "uint256" }, - { internalType: "uint256", name: "minStableOut", type: "uint256" }, - ], - name: "remove", - outputs: [ - { internalType: "uint256", name: "delRisky", type: "uint256" }, - { internalType: "uint256", name: "delStable", type: "uint256" }, - ], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "from", type: "address" }, - { internalType: "address", name: "to", type: "address" }, - { internalType: "uint256[]", name: "ids", type: "uint256[]" }, - { internalType: "uint256[]", name: "amounts", type: "uint256[]" }, - { internalType: "bytes", name: "data", type: "bytes" }, - ], - name: "safeBatchTransferFrom", - outputs: [], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "from", type: "address" }, - { internalType: "address", name: "to", type: "address" }, - { internalType: "uint256", name: "id", type: "uint256" }, - { internalType: "uint256", name: "amount", type: "uint256" }, - { internalType: "bytes", name: "data", type: "bytes" }, - ], - name: "safeTransferFrom", - outputs: [], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "token", type: "address" }, - { internalType: "uint256", name: "value", type: "uint256" }, - { internalType: "uint256", name: "deadline", type: "uint256" }, - { internalType: "uint8", name: "v", type: "uint8" }, - { internalType: "bytes32", name: "r", type: "bytes32" }, - { internalType: "bytes32", name: "s", type: "bytes32" }, - ], - name: "selfPermit", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "token", type: "address" }, - { internalType: "uint256", name: "nonce", type: "uint256" }, - { internalType: "uint256", name: "expiry", type: "uint256" }, - { internalType: "uint8", name: "v", type: "uint8" }, - { internalType: "bytes32", name: "r", type: "bytes32" }, - { internalType: "bytes32", name: "s", type: "bytes32" }, - ], - name: "selfPermitAllowed", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "token", type: "address" }, - { internalType: "uint256", name: "nonce", type: "uint256" }, - { internalType: "uint256", name: "expiry", type: "uint256" }, - { internalType: "uint8", name: "v", type: "uint8" }, - { internalType: "bytes32", name: "r", type: "bytes32" }, - { internalType: "bytes32", name: "s", type: "bytes32" }, - ], - name: "selfPermitAllowedIfNecessary", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "token", type: "address" }, - { internalType: "uint256", name: "value", type: "uint256" }, - { internalType: "uint256", name: "deadline", type: "uint256" }, - { internalType: "uint8", name: "v", type: "uint8" }, - { internalType: "bytes32", name: "r", type: "bytes32" }, - { internalType: "bytes32", name: "s", type: "bytes32" }, - ], - name: "selfPermitIfNecessary", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "operator", type: "address" }, - { internalType: "bool", name: "approved", type: "bool" }, - ], - name: "setApprovalForAll", - outputs: [], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [{ internalType: "bytes4", name: "interfaceId", type: "bytes4" }], - name: "supportsInterface", - outputs: [{ internalType: "bool", name: "", type: "bool" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [ - { - components: [ - { internalType: "address", name: "recipient", type: "address" }, - { internalType: "address", name: "risky", type: "address" }, - { internalType: "address", name: "stable", type: "address" }, - { internalType: "bytes32", name: "poolId", type: "bytes32" }, - { internalType: "bool", name: "riskyForStable", type: "bool" }, - { internalType: "uint256", name: "deltaIn", type: "uint256" }, - { internalType: "uint256", name: "deltaOut", type: "uint256" }, - { internalType: "bool", name: "fromMargin", type: "bool" }, - { internalType: "bool", name: "toMargin", type: "bool" }, - { internalType: "uint256", name: "deadline", type: "uint256" }, - ], - internalType: "struct ISwapManager.SwapParams", - name: "params", - type: "tuple", - }, - ], - name: "swap", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "uint256", name: "delRisky", type: "uint256" }, - { internalType: "uint256", name: "delStable", type: "uint256" }, - { internalType: "bytes", name: "data", type: "bytes" }, - ], - name: "swapCallback", - outputs: [], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "token", type: "address" }, - { internalType: "uint256", name: "amountMin", type: "uint256" }, - { internalType: "address", name: "recipient", type: "address" }, - ], - name: "sweepToken", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { - inputs: [ - { internalType: "uint256", name: "amountMin", type: "uint256" }, - { internalType: "address", name: "recipient", type: "address" }, - ], - name: "unwrap", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { - inputs: [{ internalType: "uint256", name: "tokenId", type: "uint256" }], - name: "uri", - outputs: [{ internalType: "string", name: "", type: "string" }], - stateMutability: "view", - type: "function", - }, - { - inputs: [ - { internalType: "address", name: "recipient", type: "address" }, - { internalType: "address", name: "engine", type: "address" }, - { internalType: "uint256", name: "delRisky", type: "uint256" }, - { internalType: "uint256", name: "delStable", type: "uint256" }, - ], - name: "withdraw", - outputs: [], - stateMutability: "nonpayable", - type: "function", - }, - { - inputs: [{ internalType: "uint256", name: "value", type: "uint256" }], - name: "wrap", - outputs: [], - stateMutability: "payable", - type: "function", - }, - { stateMutability: "payable", type: "receive" }, -] as const; diff --git a/examples/feature-api-functions/abis/erc20ABI.ts b/examples/feature-api-functions/abis/erc20ABI.ts new file mode 100644 index 000000000..94cbc6a33 --- /dev/null +++ b/examples/feature-api-functions/abis/erc20ABI.ts @@ -0,0 +1,147 @@ +export const erc20ABI = [ + { + stateMutability: "view", + type: "function", + inputs: [], + name: "DOMAIN_SEPARATOR", + outputs: [{ name: "", internalType: "bytes32", type: "bytes32" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [ + { name: "", internalType: "address", type: "address" }, + { name: "", internalType: "address", type: "address" }, + ], + name: "allowance", + outputs: [{ name: "", internalType: "uint256", type: "uint256" }], + }, + { + stateMutability: "nonpayable", + type: "function", + inputs: [ + { name: "spender", internalType: "address", type: "address" }, + { name: "amount", internalType: "uint256", type: "uint256" }, + ], + name: "approve", + outputs: [{ name: "", internalType: "bool", type: "bool" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [{ name: "", internalType: "address", type: "address" }], + name: "balanceOf", + outputs: [{ name: "", internalType: "uint256", type: "uint256" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [], + name: "decimals", + outputs: [{ name: "", internalType: "uint8", type: "uint8" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [], + name: "name", + outputs: [{ name: "", internalType: "string", type: "string" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [{ name: "", internalType: "address", type: "address" }], + name: "nonces", + outputs: [{ name: "", internalType: "uint256", type: "uint256" }], + }, + { + stateMutability: "nonpayable", + type: "function", + inputs: [ + { name: "owner", internalType: "address", type: "address" }, + { name: "spender", internalType: "address", type: "address" }, + { name: "value", internalType: "uint256", type: "uint256" }, + { name: "deadline", internalType: "uint256", type: "uint256" }, + { name: "v", internalType: "uint8", type: "uint8" }, + { name: "r", internalType: "bytes32", type: "bytes32" }, + { name: "s", internalType: "bytes32", type: "bytes32" }, + ], + name: "permit", + outputs: [], + }, + { + stateMutability: "view", + type: "function", + inputs: [], + name: "symbol", + outputs: [{ name: "", internalType: "string", type: "string" }], + }, + { + stateMutability: "view", + type: "function", + inputs: [], + name: "totalSupply", + outputs: [{ name: "", internalType: "uint256", type: "uint256" }], + }, + { + stateMutability: "nonpayable", + type: "function", + inputs: [ + { name: "to", internalType: "address", type: "address" }, + { name: "amount", internalType: "uint256", type: "uint256" }, + ], + name: "transfer", + outputs: [{ name: "", internalType: "bool", type: "bool" }], + }, + { + stateMutability: "nonpayable", + type: "function", + inputs: [ + { name: "from", internalType: "address", type: "address" }, + { name: "to", internalType: "address", type: "address" }, + { name: "amount", internalType: "uint256", type: "uint256" }, + ], + name: "transferFrom", + outputs: [{ name: "", internalType: "bool", type: "bool" }], + }, + { + type: "event", + anonymous: false, + inputs: [ + { + name: "owner", + internalType: "address", + type: "address", + indexed: true, + }, + { + name: "spender", + internalType: "address", + type: "address", + indexed: true, + }, + { + name: "amount", + internalType: "uint256", + type: "uint256", + indexed: false, + }, + ], + name: "Approval", + }, + { + type: "event", + anonymous: false, + inputs: [ + { name: "from", internalType: "address", type: "address", indexed: true }, + { name: "to", internalType: "address", type: "address", indexed: true }, + { + name: "amount", + internalType: "uint256", + type: "uint256", + indexed: false, + }, + ], + name: "Transfer", + }, +] as const; diff --git a/examples/feature-api-functions/ponder.config.ts b/examples/feature-api-functions/ponder.config.ts index 8494f8856..32ae5a2ba 100644 --- a/examples/feature-api-functions/ponder.config.ts +++ b/examples/feature-api-functions/ponder.config.ts @@ -1,7 +1,6 @@ import { createConfig } from "@ponder/core"; import { http } from "viem"; - -import { PrimitiveManagerAbi } from "./abis/PrimitiveManagerAbi"; +import { erc20ABI } from "./abis/erc20ABI"; export default createConfig({ networks: { @@ -11,11 +10,12 @@ export default createConfig({ }, }, contracts: { - PrimitiveManager: { + ERC20: { network: "mainnet", - abi: PrimitiveManagerAbi, - address: "0x54522dA62a15225C95b01bD61fF58b866C50471f", - startBlock: 14438081, + abi: erc20ABI, + address: "0x32353A6C91143bfd6C7d363B546e62a9A2489A20", + startBlock: 13142655, + endBlock: 13150000, }, }, }); diff --git a/examples/feature-api-functions/ponder.schema.ts b/examples/feature-api-functions/ponder.schema.ts index 6a3627629..a722500c6 100644 --- a/examples/feature-api-functions/ponder.schema.ts +++ b/examples/feature-api-functions/ponder.schema.ts @@ -1,9 +1,50 @@ import { createSchema } from "@ponder/core"; export default createSchema((p) => ({ - SwapEvent: p.createTable({ + Account: p.createTable({ + id: p.hex(), + balance: p.bigint(), + isOwner: p.boolean(), + + allowances: p.many("Allowance.ownerId"), + approvalOwnerEvents: p.many("ApprovalEvent.ownerId"), + approvalSpenderEvents: p.many("ApprovalEvent.spenderId"), + transferFromEvents: p.many("TransferEvent.fromId"), + transferToEvents: p.many("TransferEvent.toId"), + }), + Allowance: p.createTable({ + id: p.string(), + amount: p.bigint(), + + ownerId: p.hex().references("Account.id"), + spenderId: p.hex().references("Account.id"), + + owner: p.one("ownerId"), + spender: p.one("spenderId"), + }), + TransferEvent: p.createTable( + { + id: p.string(), + amount: p.bigint(), + timestamp: p.int(), + + fromId: p.hex().references("Account.id"), + toId: p.hex().references("Account.id"), + + from: p.one("fromId"), + to: p.one("toId"), + }, + { fromIdIndex: p.index("fromId") }, + ), + ApprovalEvent: p.createTable({ id: p.string(), - recipient: p.hex(), - payer: p.hex(), + amount: p.bigint(), + timestamp: p.int(), + + ownerId: p.hex().references("Account.id"), + spenderId: p.hex().references("Account.id"), + + owner: p.one("ownerId"), + spender: p.one("spenderId"), }), })); diff --git a/examples/feature-api-functions/src/api/index.ts b/examples/feature-api-functions/src/api/index.ts index bc24a9557..219dbde3b 100644 --- a/examples/feature-api-functions/src/api/index.ts +++ b/examples/feature-api-functions/src/api/index.ts @@ -1,21 +1,47 @@ import { ponder } from "@/generated"; -import { count, eq, or } from "@ponder/core"; -import { getAddress } from "viem"; +import { count, desc, eq, graphql, or, replaceBigInts } from "@ponder/core"; +import { formatEther, getAddress } from "viem"; + +ponder.use("/graphql", graphql()); ponder.get("/count", async (c) => { - const result = await c.db.select({ count: count() }).from(c.tables.SwapEvent); + const result = await c.db + .select({ count: count() }) + .from(c.tables.TransferEvent); - return c.text(String(result[0]?.count ?? 0)); + if (result.length === 0) return c.text("0"); + return c.text(String(result[0]!.count)); }); -ponder.get("/user-count/:address", async (c) => { +ponder.get("/count/:address", async (c) => { const account = getAddress(c.req.param("address")); - const { SwapEvent } = c.tables; + const { TransferEvent } = c.tables; const result = await c.db .select({ count: count() }) - .from(c.tables.SwapEvent) - .where(or(eq(SwapEvent.payer, account), eq(SwapEvent.recipient, account))); + .from(c.tables.TransferEvent) + .where( + or(eq(TransferEvent.fromId, account), eq(TransferEvent.toId, account)), + ); + + if (result.length === 0) return c.text("0"); + return c.text(String(result[0]!.count)); +}); + +ponder.get("/whale-transfers", async (c) => { + const { TransferEvent, Account } = c.tables; + + // Top 10 transfers from whale accounts + const result = await c.db + .select({ + amount: TransferEvent.amount, + senderBalance: Account.balance, + }) + .from(TransferEvent) + .innerJoin(Account, eq(TransferEvent.fromId, Account.id)) + .orderBy(desc(Account.balance)) + .limit(10); - return c.text(String(result[0]?.count ?? 0)); + if (result.length === 0) return c.text("Not found", 500); + return c.json(replaceBigInts(result, (b) => formatEther(b))); }); diff --git a/examples/feature-api-functions/src/index.ts b/examples/feature-api-functions/src/index.ts index ce46fbbcb..16bf33aa7 100644 --- a/examples/feature-api-functions/src/index.ts +++ b/examples/feature-api-functions/src/index.ts @@ -1,13 +1,70 @@ import { ponder } from "@/generated"; -ponder.on("PrimitiveManager:Swap", async ({ event, context }) => { - const { SwapEvent } = context.db; +ponder.on("ERC20:Transfer", async ({ event, context }) => { + const { Account, TransferEvent } = context.db; - await SwapEvent.create({ + // Create an Account for the sender, or update the balance if it already exists. + await Account.upsert({ + id: event.args.from, + create: { + balance: BigInt(0), + isOwner: false, + }, + update: ({ current }) => ({ + balance: current.balance - event.args.amount, + }), + }); + + // Create an Account for the recipient, or update the balance if it already exists. + await Account.upsert({ + id: event.args.to, + create: { + balance: event.args.amount, + isOwner: false, + }, + update: ({ current }) => ({ + balance: current.balance + event.args.amount, + }), + }); + + // Create a TransferEvent. + await TransferEvent.create({ + id: event.log.id, + data: { + fromId: event.args.from, + toId: event.args.to, + amount: event.args.amount, + timestamp: Number(event.block.timestamp), + }, + }); +}); + +ponder.on("ERC20:Approval", async ({ event, context }) => { + const { Allowance, ApprovalEvent } = context.db; + + const allowanceId = `${event.args.owner}-${event.args.spender}`; + + // Create or update the Allowance. + await Allowance.upsert({ + id: allowanceId, + create: { + ownerId: event.args.owner, + spenderId: event.args.spender, + amount: event.args.amount, + }, + update: { + amount: event.args.amount, + }, + }); + + // Create an ApprovalEvent. + await ApprovalEvent.create({ id: event.log.id, data: { - payer: event.args.payer, - recipient: event.args.recipient, + ownerId: event.args.owner, + spenderId: event.args.spender, + amount: event.args.amount, + timestamp: Number(event.block.timestamp), }, }); }); From 4012209694abd91f037ad480d9797a6aa2a6d73b Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:31:16 -0400 Subject: [PATCH 118/122] docs --- .../docs/indexing/create-update-records.mdx | 4 +- docs/pages/docs/migration-guide.mdx | 36 ++++- docs/pages/docs/query/_meta.ts | 2 +- docs/pages/docs/query/api-functions.mdx | 144 +++++++++++++----- 4 files changed, 144 insertions(+), 42 deletions(-) diff --git a/docs/pages/docs/indexing/create-update-records.mdx b/docs/pages/docs/indexing/create-update-records.mdx index 8585da6fb..6876c8132 100644 --- a/docs/pages/docs/indexing/create-update-records.mdx +++ b/docs/pages/docs/indexing/create-update-records.mdx @@ -1,11 +1,11 @@ --- -title: "Create and Update Records" +title: "Create and update records" description: "Learn how to create and update records in the Ponder database." --- # Create & update records -Ponder's store API is inspired by the [Prisma Client API](https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#model-queries). The store supports the following methods. +The **Store API** is inspired by [Prisma](https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#model-queries) and supports the following methods. - [`create`](#create) - [`update`](#update) diff --git a/docs/pages/docs/migration-guide.mdx b/docs/pages/docs/migration-guide.mdx index e5ad2e3ea..c5c6c12ea 100644 --- a/docs/pages/docs/migration-guide.mdx +++ b/docs/pages/docs/migration-guide.mdx @@ -3,12 +3,38 @@ title: "Migration Guide" description: "A guide for migrating to new versions of Ponder." --- -import { FileTree, Steps } from "nextra/components"; -import { Callout } from "nextra/components"; +import { FileTree, Steps, Tabs, Callout } from "nextra/components"; import Architecture from "../../public/architecture.svg"; # Migration guide +## 0.5.0 + +**Breaking:** This release adds [Hono](https://hono.dev) as a peer dependency. After upgrading, install `hono` in your project. + +{/* prettier-ignore */} + + +```bash filename="shell" +pnpm add hono +``` + + +```bash filename="shell" +yarn add hono +``` + + +```bash filename="shell" +npm install hono +``` + + + +### Introduced API functions + +This release added support for API functions. [Read more](/docs/query/api-functions). + ## 0.4.0 This release changes the location of database tables when using both SQLite and Postgres. @@ -17,11 +43,11 @@ It **does not** require any changes to your application code, and does not bust Please read the new docs on [direct SQL](/docs/query/direct-sql) for a detailed overview. -#### SQLite +### SQLite Ponder now uses the `.ponder/sqlite/public.db` file for indexed tables. Before, the tables were present as views in the `.ponder/sqlite/ponder.db`. Now, the`.ponder/sqlite/ponder.db` file is only used internally by Ponder. -#### Postgres +### Postgres Ponder now creates a table in the `public` schema for each table in `ponder.schema.ts`. Before, Ponder created them as views in the `ponder` schema. @@ -31,7 +57,7 @@ This also changes the zero-downtime behavior on platforms like Railway. For more - [Direct SQL](/docs/query/direct-sql) - [Zero-downtime deployments](/docs/production/zero-downtime) -#### Postgres table cleanup +### Postgres table cleanup After upgrading to `0.4.x`, you can run the following Postgres SQL script to clean up stale tables and views created by `0.3.x` Ponder apps. diff --git a/docs/pages/docs/query/_meta.ts b/docs/pages/docs/query/_meta.ts index 3e5b32faf..49e84a20c 100644 --- a/docs/pages/docs/query/_meta.ts +++ b/docs/pages/docs/query/_meta.ts @@ -1,5 +1,5 @@ export default { - "api-functions": "API functions (experimental)", + "api-functions": "API functions", "graphql": "GraphQL", "direct-sql": "Direct SQL", }; diff --git a/docs/pages/docs/query/api-functions.mdx b/docs/pages/docs/query/api-functions.mdx index 9fca08c2d..e4acbfcde 100644 --- a/docs/pages/docs/query/api-functions.mdx +++ b/docs/pages/docs/query/api-functions.mdx @@ -5,9 +5,9 @@ description: "Use API functions to customize the API layer of your app." import { Callout, Steps } from "nextra/components"; -# API functions (experimental) +# API functions -**API functions** are user-defined functions that respond to incoming Web requests. You can use them to customize the API layer of your app with direct SQL queries, authentication, data from external sources, and more. +**API functions** are user-defined TypeScript functions that handle web requests. You can use them to customize the API layer of your app with complex SQL queries, authentication, data from external sources, and more. API functions are built on top of [Hono](https://hono.dev/), a fast and lightweight routing framework. @@ -15,17 +15,13 @@ API functions are built on top of [Hono](https://hono.dev/), a fast and lightwei -### Install `hono` and `@ponder/core@next` +### Upgrade to `>=0.5.0` -To use API functions, install `hono` and the prerelease version of `@ponder/core`. - -```bash -npm install hono @ponder/core@next -``` +API functions are available starting from version `0.5.0`. Read the [migration guide](/docs/migration-guide#050) for more details. ### Create `src/api/index.ts` file -To opt-in to API functions, create a file named `src/api/index.ts` with the following code. +To enable API functions, create a file named `src/api/index.ts` with the following code. ```ts filename="src/api/index.ts" import { ponder } from "@/generated"; @@ -37,16 +33,26 @@ ponder.get("/hello", (c) => { ### Send a request -With the server running, visit `http://localhost:42069/hello` in your browser to see the response. +Visit `http://localhost:42069/hello` in your browser to see the response. + +```plaintext filename="Response" +Hello, world! +``` + +### Register GraphQL middleware -### Add the GraphQL middleware + + Once you create a file in the `src/api/` directory, Ponder stops serving the + standard GraphQL API. + -To register the standard GraphQL API that was automatically included in versions `<0.5.0`, register the `graphql` middleware exported from `@ponder/core`. +To continue using the standard GraphQL API, register the `graphql` middleware exported from `@ponder/core`. -```ts filename="src/api/index.ts" {2,4} +```ts filename="src/api/index.ts" {2,4-5} import { ponder } from "@/generated"; import { graphql } from "@ponder/core"; +ponder.use("/", graphql()); ponder.use("/graphql", graphql()); // ... @@ -56,22 +62,26 @@ ponder.use("/graphql", graphql()); ## Query the database -API functions have read-only access to your database using a [Hono Variable](https://hono.dev/docs/api/context#set-get) named `db`. Unlike indexing functions, the `create`, `update`, `delete`, `createMany`, and `updateMany` methods are not available in API functions. +API functions can query the database using the read-only **Select API**, a type-safe query builder powered by [Drizzle](https://orm.drizzle.team/docs/overview). The Select API supports complex filters, joins, aggregations, set operations, and more. + + + The Select API is only available within API functions. Indexing functions use + the [Store API](/docs/indexing/create-update-records) (`findUnique`, `upsert`, + etc) which supports writes and is reorg-aware. + -### `findUnique` and `findMany` +### Select -API functions can use `findUnique` and `findMany` to query the database. These methods work just like they do within indexing functions. [Read more](/docs/indexing/create-update-records#findmany) about the store API. +Use `c.db` and `c.tables` to build custom queries using Drizzle's [Select](https://orm.drizzle.team/docs/select) query builder syntax. ```ts filename="src/api/index.ts" {5,7-9} import { ponder } from "@/generated"; -ponder.get("/account/:address", (c) => { +ponder.get("/account/:address", async (c) => { + const { Account, TransferEvent } = c.tables; const address = c.req.param("address"); - const db = c.get("db"); - const account = await db.Account.findUnique({ - id: address, - }); + const account = await c.db.select(account).where({ id: address }).first(); if (account) { return c.json(account); @@ -81,19 +91,18 @@ ponder.get("/account/:address", (c) => { }); ``` -### Direct SQL +### Execute -Use `db.query(...){:ts}` to execute raw SQL queries against your database. +Run raw SQL queries with `db.execute(...){:ts}` and the `sql` helper function. ```ts filename="src/api/index.ts" {6,8-10} import { ponder } from "@/generated"; import { sql } from "@ponder/core"; -ponder.get("/:token/ticker", (c) => { +ponder.get("/:token/ticker", async (c) => { const token = c.req.param("token"); - const db = c.get("db"); - const result = await db.query( + const result = await c.db.execute( sql`SELECT ticker FROM "Token" WHERE id = ${token}` ); const ticker = result.rows[0]?.ticker; @@ -102,9 +111,76 @@ ponder.get("/:token/ticker", (c) => { }); ``` -## Register middleware +## API reference + +### `ponder.get()` + +Use `ponder.get()` to handle HTTP `GET` requests. The context (`c`) contains the request object, response helpers, and the database connection. + +```ts filename="src/api/index.ts" {5,7-9} +import { ponder } from "@/generated"; + +ponder.get("/account/:address", async (c) => { + const { Account } = c.tables; + const address = c.req.param("address"); + + const account = await c.db + .select() + .from(Account) + .where({ id: address }) + .first(); + + if (account) { + return c.json(account); + } else { + return c.status(404).json({ error: "Account not found" }); + } +}); +``` + +### `ponder.post()` + +Use `ponder.post()` to handle HTTP `POST` requests. + + + API functions cannot write to the database, even when handling `POST` + requests. + + +In this example, we calculate the volume of transfers for each recipient within a given time range. The `fromTimestamp` and `toTimestamp` parameters are passed in the request body. + +```ts filename="src/api/index.ts" {5,7-9} +import { ponder } from "@/generated"; +import { gte } from "@ponder/core"; + +ponder.post("/volume", async (c) => { + const { TransferEvent } = c.tables; + + const body = await c.req.json(); + const { fromTimestamp, toTimestamp } = body; + + const volumeChartData = await c.db + .select({ + to: TransferEvent.toId, + volume: sum(TransferEvent.amount), + }) + .from(TransferEvent) + .groupBy(TransferEvent.toId) + .where( + and( + gte(TransferEvent.timestamp, fromTimestamp), + lte(TransferEvent.timestamp, toTimestamp) + ) + ) + .limit(1); + + return c.json(volumeChartData); +}); +``` + +### `ponder.use()` -Use `ponder.use(...){:ts}` to add middleware to your API functions. Middleware functions can modify the request and response objects, add logs, do authentication, and more. [Read more](https://hono.dev/docs/guides/middleware) about Hono middleware. +Use `ponder.use(...){:ts}` to add middleware to your API functions. Middleware functions can modify the request and response objects, add logs, authenticate requests, and more. [Read more](https://hono.dev/docs/guides/middleware) about Hono middleware. ```ts filename="src/api/index.ts" {3} import { ponder } from "@/generated"; @@ -115,9 +191,9 @@ ponder.use((c) => { }); ``` -## Access the Hono instance +### `ponder.hono` -You can register API functions and middleware using `ponder.get(...){:ts}`, `ponder.post(...){:ts}`, and `ponder.use(...){:ts}`. If you need to access the underlying Hono instance, use the `hono` property. +If you need to access the underlying Hono instance, use the `hono` property. ```ts filename="src/api/index.ts" {3} import { ponder } from "@/generated"; @@ -129,9 +205,9 @@ ponder.hono.notFound((c) => { // ... ``` -## Internal routes +## Reserved routes -Ponder registers internal routes under the `/_ponder/*` path. If you attempt to register API functions that conflict with internal routes, the build will fail. +If you register API functions that conflict with these internal routes, the build will fail. -- `/_ponder/health`: Returns a `200` status code after the app has completed historical indexing OR the healthcheck timeout has expired, whichever comes first. [Read more](/docs/production/zero-downtime) about healthchecks. -- `/_ponder/metrics`: Returns Prometheus metrics. [Read more](/docs/advanced/metrics) about metrics. +- `/health`: Returns a `200` status code after the app has completed historical indexing OR the healthcheck timeout has expired, whichever comes first. [Read more](/docs/production/zero-downtime) about healthchecks. +- `/metrics`: Returns Prometheus metrics. [Read more](/docs/advanced/metrics) about metrics. From b7e5016e06e98f238369685d06f38e5c1ea80a5b Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Wed, 17 Jul 2024 15:50:09 -0400 Subject: [PATCH 119/122] add hono and new env to create ponder --- packages/create-ponder/templates/empty/package.json | 1 + packages/create-ponder/templates/empty/ponder-env.d.ts | 1 + packages/create-ponder/templates/etherscan/package.json | 1 + packages/create-ponder/templates/etherscan/ponder-env.d.ts | 1 + packages/create-ponder/templates/subgraph/package.json | 1 + packages/create-ponder/templates/subgraph/ponder-env.d.ts | 1 + 6 files changed, 6 insertions(+) diff --git a/packages/create-ponder/templates/empty/package.json b/packages/create-ponder/templates/empty/package.json index 083cd2e68..0b6368412 100644 --- a/packages/create-ponder/templates/empty/package.json +++ b/packages/create-ponder/templates/empty/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "^0.0.95", + "hono": "^4.5.0", "viem": "^1.19.3" }, "devDependencies": { diff --git a/packages/create-ponder/templates/empty/ponder-env.d.ts b/packages/create-ponder/templates/empty/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/packages/create-ponder/templates/empty/ponder-env.d.ts +++ b/packages/create-ponder/templates/empty/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/packages/create-ponder/templates/etherscan/package.json b/packages/create-ponder/templates/etherscan/package.json index 94e7e0c44..c69bab9ea 100644 --- a/packages/create-ponder/templates/etherscan/package.json +++ b/packages/create-ponder/templates/etherscan/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "^0.0.95", + "hono": "^4.5.0", "viem": "^1.19.3" }, "devDependencies": { diff --git a/packages/create-ponder/templates/etherscan/ponder-env.d.ts b/packages/create-ponder/templates/etherscan/ponder-env.d.ts index f8e7347cf..03126bf92 100644 --- a/packages/create-ponder/templates/etherscan/ponder-env.d.ts +++ b/packages/create-ponder/templates/etherscan/ponder-env.d.ts @@ -21,6 +21,7 @@ declare module "@/generated" { schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = Virtual.IndexingFunctionArgs; export type Schema = Virtual.Schema; diff --git a/packages/create-ponder/templates/subgraph/package.json b/packages/create-ponder/templates/subgraph/package.json index 94e7e0c44..c69bab9ea 100644 --- a/packages/create-ponder/templates/subgraph/package.json +++ b/packages/create-ponder/templates/subgraph/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@ponder/core": "^0.0.95", + "hono": "^4.5.0", "viem": "^1.19.3" }, "devDependencies": { diff --git a/packages/create-ponder/templates/subgraph/ponder-env.d.ts b/packages/create-ponder/templates/subgraph/ponder-env.d.ts index 2f5363e0c..1169bd3f1 100644 --- a/packages/create-ponder/templates/subgraph/ponder-env.d.ts +++ b/packages/create-ponder/templates/subgraph/ponder-env.d.ts @@ -25,6 +25,7 @@ declare module "@/generated" { Schema, name >; + export type ApiContext = Virtual.Drizzle; export type IndexingFunctionArgs = { event: Event; context: Context; From 5cab628262da5b6ff0553469f69d9c02f3c23584 Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Wed, 17 Jul 2024 17:52:43 -0400 Subject: [PATCH 120/122] docs --- docs/pages/docs/migration-guide.mdx | 6 +- docs/pages/docs/query/api-functions.mdx | 83 ++++++++++++++++++------- 2 files changed, 65 insertions(+), 24 deletions(-) diff --git a/docs/pages/docs/migration-guide.mdx b/docs/pages/docs/migration-guide.mdx index c5c6c12ea..297d35a38 100644 --- a/docs/pages/docs/migration-guide.mdx +++ b/docs/pages/docs/migration-guide.mdx @@ -16,17 +16,17 @@ import Architecture from "../../public/architecture.svg"; ```bash filename="shell" -pnpm add hono +pnpm add hono@latest ``` ```bash filename="shell" -yarn add hono +yarn add hono@latest ``` ```bash filename="shell" -npm install hono +npm install hono@latest ``` diff --git a/docs/pages/docs/query/api-functions.mdx b/docs/pages/docs/query/api-functions.mdx index e4acbfcde..00172b0f9 100644 --- a/docs/pages/docs/query/api-functions.mdx +++ b/docs/pages/docs/query/api-functions.mdx @@ -11,6 +11,13 @@ import { Callout, Steps } from "nextra/components"; API functions are built on top of [Hono](https://hono.dev/), a fast and lightweight routing framework. +## Example projects + +These example apps demonstrate how to use API functions. + +- [**Basic**](https://github.com/ponder-sh/ponder/tree/main/examples/feature-api-functions/src/api/index.ts) - An ERC20 app that responds to `GET` requests and uses the [Select API](#query-the-database) to build custom SQL queries. +- [**tRPC**](https://github.com/ponder-sh/ponder/tree/main/examples/with-trpc/client/index.ts) - An app that creates a [tRPC](https://trpc.io) server and a script that uses a tRPC client with end-to-end type safety. + ## Get started @@ -21,7 +28,7 @@ API functions are available starting from version `0.5.0`. Read the [migration g ### Create `src/api/index.ts` file -To enable API functions, create a file named `src/api/index.ts` with the following code. +To enable API functions, create a file named `src/api/index.ts` with the following code. You can register API functions in any `.ts` file in the `src/api/` directory. ```ts filename="src/api/index.ts" import { ponder } from "@/generated"; @@ -41,9 +48,9 @@ Hello, world! ### Register GraphQL middleware - - Once you create a file in the `src/api/` directory, Ponder stops serving the - standard GraphQL API. + + Once you create an API function file, you have "opted in" to API functions and + your app **will not** serve the standard GraphQL API by default. To continue using the standard GraphQL API, register the `graphql` middleware exported from `@ponder/core`. @@ -72,30 +79,63 @@ API functions can query the database using the read-only **Select API**, a type- ### Select -Use `c.db` and `c.tables` to build custom queries using Drizzle's [Select](https://orm.drizzle.team/docs/select) query builder syntax. +The API function context contains a built-in database client (`db`) and an object for each table in your schema (`tables`). These objects are type-safe – changes to your `ponder.schema.ts` file will be reflected immediately. -```ts filename="src/api/index.ts" {5,7-9} +To build a query, use `c.db.select(){:ts}` and include a table object using `.from(c.tables.TableName){:ts}`. + +
+ +```ts filename="ponder.schema.ts" {4} +import { createSchema } from "@ponder/core"; + +export default createSchema((p) => ({ + Account: p.createTable({ + id: p.string(), + balance: p.bigint(), + }), +})); +``` + +```ts filename="src/api/index.ts" {6} import { ponder } from "@/generated"; ponder.get("/account/:address", async (c) => { - const { Account, TransferEvent } = c.tables; const address = c.req.param("address"); - const account = await c.db.select(account).where({ id: address }).first(); + const account = await c.db.select(c.tables.Account).limit(1); - if (account) { - return c.json(account); - } else { - return c.status(404).json({ error: "Account not found" }); - } + return c.json(account); +}); +``` + +
+ +To build more complex queries, use `join`, `groupBy`, `where`, `orderBy`, `limit`, and other methods. Drizzle's filter & conditional operators (like `eq`, `gte`, and `or`) are re-exported by `@ponder/core`. + +For more details, please reference the [Drizzle documentation](https://orm.drizzle.team/docs/select). + +```ts filename="src/api/index.ts" {2,7-11} +import { ponder } from "@/generated"; +import { gte } from "@ponder/core"; + +ponder.get("/whales", async (c) => { + const { Account } = c.tables; + + const whales = await c.db + .select({ address: Account.id, balance: Account.balance }) + .from(Account.id) + .where(gte(TransferEvent.balance, 1_000_000_000n)) + .limit(1); + + return c.json(whales); }); ``` ### Execute -Run raw SQL queries with `db.execute(...){:ts}` and the `sql` helper function. +To run raw SQL queries, use `db.execute(...){:ts}` with the `sql` utility function. [Read more](https://orm.drizzle.team/docs/sql) about the `sql` function. -```ts filename="src/api/index.ts" {6,8-10} +```ts filename="src/api/index.ts" {2,7-9} import { ponder } from "@/generated"; import { sql } from "@ponder/core"; @@ -115,9 +155,9 @@ ponder.get("/:token/ticker", async (c) => { ### `ponder.get()` -Use `ponder.get()` to handle HTTP `GET` requests. The context (`c`) contains the request object, response helpers, and the database connection. +Use `ponder.get()` to handle HTTP `GET` requests. The `c` context object contains the request, response helpers, and the database connection. -```ts filename="src/api/index.ts" {5,7-9} +```ts filename="src/api/index.ts" {3,5,7-9} import { ponder } from "@/generated"; ponder.get("/account/:address", async (c) => { @@ -140,18 +180,18 @@ ponder.get("/account/:address", async (c) => { ### `ponder.post()` -Use `ponder.post()` to handle HTTP `POST` requests. - - + API functions cannot write to the database, even when handling `POST` requests. +Use `ponder.post()` to handle HTTP `POST` requests. + In this example, we calculate the volume of transfers for each recipient within a given time range. The `fromTimestamp` and `toTimestamp` parameters are passed in the request body. ```ts filename="src/api/index.ts" {5,7-9} import { ponder } from "@/generated"; -import { gte } from "@ponder/core"; +import { and, gte, sum } from "@ponder/core"; ponder.post("/volume", async (c) => { const { TransferEvent } = c.tables; @@ -211,3 +251,4 @@ If you register API functions that conflict with these internal routes, the buil - `/health`: Returns a `200` status code after the app has completed historical indexing OR the healthcheck timeout has expired, whichever comes first. [Read more](/docs/production/zero-downtime) about healthchecks. - `/metrics`: Returns Prometheus metrics. [Read more](/docs/advanced/metrics) about metrics. +- `/status`: Returns indexing status object. [Read more](/docs/advanced/status) about indexing status. From 1c539bc6731d5c91a4eac234bd16a04ba50fd796 Mon Sep 17 00:00:00 2001 From: typedarray <90073088+0xOlias@users.noreply.github.com> Date: Wed, 17 Jul 2024 17:54:53 -0400 Subject: [PATCH 121/122] tweaks --- docs/pages/docs/query/api-functions.mdx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/pages/docs/query/api-functions.mdx b/docs/pages/docs/query/api-functions.mdx index 00172b0f9..f18d13679 100644 --- a/docs/pages/docs/query/api-functions.mdx +++ b/docs/pages/docs/query/api-functions.mdx @@ -153,11 +153,11 @@ ponder.get("/:token/ticker", async (c) => { ## API reference -### `ponder.get()` +### `get()` Use `ponder.get()` to handle HTTP `GET` requests. The `c` context object contains the request, response helpers, and the database connection. -```ts filename="src/api/index.ts" {3,5,7-9} +```ts filename="src/api/index.ts" {3,5} import { ponder } from "@/generated"; ponder.get("/account/:address", async (c) => { @@ -178,7 +178,7 @@ ponder.get("/account/:address", async (c) => { }); ``` -### `ponder.post()` +### `post()` API functions cannot write to the database, even when handling `POST` @@ -218,7 +218,7 @@ ponder.post("/volume", async (c) => { }); ``` -### `ponder.use()` +### `use()` Use `ponder.use(...){:ts}` to add middleware to your API functions. Middleware functions can modify the request and response objects, add logs, authenticate requests, and more. [Read more](https://hono.dev/docs/guides/middleware) about Hono middleware. @@ -231,9 +231,9 @@ ponder.use((c) => { }); ``` -### `ponder.hono` +### `hono` -If you need to access the underlying Hono instance, use the `hono` property. +Use `ponder.hono` to access the underlying Hono instance. ```ts filename="src/api/index.ts" {3} import { ponder } from "@/generated"; From b419b846da7114cca936a6a3b404ca165572c7f2 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Thu, 18 Jul 2024 12:56:46 -0400 Subject: [PATCH 122/122] chore: changeset --- .changeset/shy-donuts-battle.md | 2 +- .changeset/silent-walls-give.md | 5 ----- packages/core/src/graphql/index.ts | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 .changeset/silent-walls-give.md diff --git a/.changeset/shy-donuts-battle.md b/.changeset/shy-donuts-battle.md index 6cbd50291..4cbc9f011 100644 --- a/.changeset/shy-donuts-battle.md +++ b/.changeset/shy-donuts-battle.md @@ -3,4 +3,4 @@ "@ponder/core": minor --- -[Experimental] Introduced API functions. [Read more](https://ponder-docs-git-kjs-api2-ponder-sh.vercel.app/docs/query/api-functions). +Introduced API functions. [Read more](https://ponder.sh/docs/query/api-functions). Please read the [migration guide](https://ponder.sh/docs/migration-guide) for more information. diff --git a/.changeset/silent-walls-give.md b/.changeset/silent-walls-give.md deleted file mode 100644 index a5053d66a..000000000 --- a/.changeset/silent-walls-give.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@ponder/core": patch ---- - -Added per network progress to the "/status" endpoint on the server and the "_metadata" entity on the GraphQL schema. diff --git a/packages/core/src/graphql/index.ts b/packages/core/src/graphql/index.ts index 1c0a296b7..bdabd7650 100644 --- a/packages/core/src/graphql/index.ts +++ b/packages/core/src/graphql/index.ts @@ -10,7 +10,7 @@ import { buildLoaderCache } from "./buildLoaderCache.js"; /** * Middleware for GraphQL with an interactive web view. * - * - Docs: [TODO(kyle)] + * - Docs: https://ponder.sh/docs/query/api-functions#register-graphql-middleware * * @example * import { ponder } from "@/generated";