diff --git a/src/bundler/fs.ts b/src/bundler/fs.ts index b4cd339..ff60154 100644 --- a/src/bundler/fs.ts +++ b/src/bundler/fs.ts @@ -54,14 +54,6 @@ function warnCrossFilesystem(dstPath: string) { export interface Filesystem { listDir(dirPath: string): Dirent[]; - // Filtered version of `listDir` that doesn't take a read dependency - // on entries filtered out. Note that `fileFilter` only applies to - // regular files, not directories. - listDirFiltered( - dirPath: string, - fileFilter: (absPath: string) => boolean, - ): Dirent[]; - exists(path: string): boolean; stat(path: string): Stats; readUtf8File(path: string): string; @@ -112,21 +104,6 @@ class NodeFs implements Filesystem { listDir(dirPath: string) { return stdFs.readdirSync(dirPath, { withFileTypes: true }); } - listDirFiltered( - dirPath: string, - fileFilter: (absPath: string) => boolean, - ): stdFs.Dirent[] { - const entries = stdFs.readdirSync(dirPath, { withFileTypes: true }); - const filtered = []; - for (const entry of entries) { - const absPath = path.join(dirPath, entry.name); - if (entry.isFile() && !fileFilter(absPath)) { - continue; - } - filtered.push(entry); - } - return filtered; - } exists(path: string) { try { stdFs.statSync(path); @@ -276,55 +253,6 @@ export class RecordingFs implements Filesystem { return entries; } - listDirFiltered( - dirPath: string, - fileFilter: (absPath: string) => boolean, - ): stdFs.Dirent[] { - const absDirPath = path.resolve(dirPath); - - // Register observing the directory itself. - const dirSt = nodeFs.stat(absDirPath); - this.registerNormalized(absDirPath, dirSt); - - // List the directory, collecting all of its child entries. - const allEntries = nodeFs.listDir(dirPath); - - // Filter the entry list, throwing out files that don't pass `fileFilter`. - // Register observing the filtered children. - const filteredEntries = []; - for (const entry of allEntries) { - const childPath = path.join(absDirPath, entry.name); - if (entry.isFile() && !fileFilter(childPath)) { - continue; - } - const childSt = nodeFs.stat(childPath); - this.registerPath(childPath, childSt); - filteredEntries.push(entry); - } - - // Register observing the directory's children. Note that - // we use the full entry list here, not the filtered list, - // since we want this observation to be shared across - // `listDirFiltered` calls with different `fileFilter`s. - const observedNames = new Set(allEntries.map((e) => e.name)); - const existingNames = this.observedDirectories.get(absDirPath); - if (existingNames) { - if (!setsEqual(observedNames, existingNames)) { - if (this.traceEvents) { - console.log( - "Invalidating due to directory children mismatch", - observedNames, - existingNames, - ); - } - this.invalidated = true; - } - } - this.observedDirectories.set(absDirPath, observedNames); - - return filteredEntries; - } - exists(path: string): boolean { try { const st = nodeFs.stat(path); diff --git a/src/cli/lib/components.ts b/src/cli/lib/components.ts index 8d7e64d..1653bdd 100644 --- a/src/cli/lib/components.ts +++ b/src/cli/lib/components.ts @@ -14,7 +14,6 @@ import { bundleDefinitions, bundleImplementations, componentGraph, - // TODO probably just a } from "./components/definition/bundle.js"; import { isComponentDirectory } from "./components/definition/directoryStructure.js"; @@ -42,13 +41,6 @@ export async function runComponentsPush(ctx: Context, options: PushOptions) { const convexDir = functionsDir(configPath, projectConfig); - // TODO - // Note we need to restart this whole process if any of these files change! - // We should track these separately if we can: if any thing observed - // by the definition traversal changes we need to restart at an earlier - // point than if something in one of the implementations changes. - // If one of the implementions changes we should be able to just rebuild that. - // '.' means use the process current working directory, it's the default behavior. // Spelling it out here to be explicit for a future where this code can run // from other directories. @@ -90,12 +82,10 @@ export async function runComponentsPush(ctx: Context, options: PushOptions) { absWorkingDir, dependencyGraph, rootComponent, - // note this *includes* the root component (TODO update bundleImpls to work this way too) + // Note that this *includes* the root component. [...components.values()], ); - // Is this possible to run in this world? - // Note that it bundles!!! That's a step we don't need. const { config: localConfig } = await configFromProjectConfig( ctx, projectConfig, @@ -111,8 +101,6 @@ export async function runComponentsPush(ctx: Context, options: PushOptions) { verbose, ); - // This expects an auth (get it the normal way) - // and functions which are pretty normal? const appDefinition: AppDefinitionSpec = { ...appDefinitionSpecWithoutImpls, auth: localConfig.authConfig || null, @@ -149,7 +137,7 @@ export async function runComponentsPush(ctx: Context, options: PushOptions) { options.adminKey, options.url, projectConfig.functions, // this is where the convex folder is, just 'convex/' - udfServerVersion, // this comes from config? + udfServerVersion, appDefinition, componentDefinitions, ); @@ -163,9 +151,4 @@ export async function runComponentsPush(ctx: Context, options: PushOptions) { startPushResponse, ); console.log("finishPush:", finishPushResponse); - - // TODO - // How to make this re-entrant? If there's a change you should be able to stop the current - // deploy and restart. If one component deep in the chain changes, you should be able. - // Cached results should be able to be used. } diff --git a/src/cli/lib/components/constants.ts b/src/cli/lib/components/constants.ts index 597e2d3..9232a90 100644 --- a/src/cli/lib/components/constants.ts +++ b/src/cli/lib/components/constants.ts @@ -1,4 +1,2 @@ export const ROOT_DEFINITION_FILENAME = "app.config.ts"; export const DEFINITION_FILENAME = "component.config.ts"; - -export const IMPLICIT_EVALUATION_RESULT = "component_data.json"; diff --git a/src/cli/lib/components/definition/bundle.ts b/src/cli/lib/components/definition/bundle.ts index 9af51a8..467f200 100644 --- a/src/cli/lib/components/definition/bundle.ts +++ b/src/cli/lib/components/definition/bundle.ts @@ -1,4 +1,3 @@ -// TODO read this import path from "path"; import { ComponentDirectory, @@ -14,33 +13,31 @@ import { logMessage, logWarning, } from "../../../../bundler/context.js"; -import esbuild, { Metafile, OutputFile, Plugin } from "esbuild"; +import esbuild, { BuildOptions, Metafile, OutputFile, Plugin } from "esbuild"; import chalk from "chalk"; -import { DEFINITION_FILENAME } from "../constants.js"; import { createRequire } from "module"; +import { + AppDefinitionSpecWithoutImpls, + ComponentDefinitionSpecWithoutImpls, +} from "../../config.js"; import { Bundle, bundle, bundleSchema, entryPointsByEnvironment, } from "../../../../bundler/index.js"; -import { - AppDefinitionSpecWithoutImpls, - ComponentDefinitionSpecWithoutImpls, -} from "../../deploy2.js"; /** - * esbuild plugin to mark component definitions external or return a list of + * An esbuild plugin to mark component definitions external or return a list of * all component definitions. * - * By default this plugin marks component definition files as external, - * not traversing further. + * By default this plugin runs in "bundle" mode and marks all imported component + * definition files as external, not traversing further. * - * If discover is specified it instead populates the components map, - * continuing through the entire tree. + * If "discover" mode is specified it traverses the entire tree. */ function componentPlugin({ - mode, + mode = "bundle", rootComponentDirectory, verbose, ctx, @@ -56,127 +53,174 @@ function componentPlugin({ async setup(build) { // This regex can't be really precise since developers could import // "component.config", "component.config.js", "component.config.ts", etc. - build.onResolve( - // Will it be important when developers write npm package components - // for these to use a specific entry point or specific file name? - // I guess we can read the files from the filesystem ourselves! - // If developers want to import the component definition directly - // from somewhere else, - { filter: /.*component.config.*/ }, - async (args) => { - verbose && logMessage(ctx, "esbuild resolving import:", args); - if (args.namespace !== "file") { - verbose && logMessage(ctx, " Not a file."); - return; - } - if (args.kind === "entry-point") { - verbose && logMessage(ctx, " -> Top-level entry-point."); - const componentDirectory = { - name: path.basename(path.dirname(args.path)), - path: args.path, - definitionPath: path.join(args.path, DEFINITION_FILENAME), - }; - if (components.get(args.path)) { - // programmer error - // eslint-disable-next-line no-restricted-syntax - throw new Error( - "why is the entry point component already registered?", - ); - } - components.set(args.path, componentDirectory); - // For an entry point there's no resolution logic to puzzle through - // since we already have a proper file path. - // Whether we're bundling or discovering, we're done. - return; - } - const candidates = [args.path]; - const ext = path.extname(args.path); - if (ext === ".js") { - candidates.push(args.path.slice(0, -".js".length) + ".ts"); - } - if (ext !== ".js" && ext !== ".ts") { - candidates.push(args.path + ".js"); - candidates.push(args.path + ".ts"); + build.onResolve({ filter: /.*component.config.*/ }, async (args) => { + verbose && logMessage(ctx, "esbuild resolving import:", args); + if (args.namespace !== "file") { + verbose && logMessage(ctx, " Not a file."); + return; + } + if (args.kind === "entry-point") { + verbose && logMessage(ctx, " -> Top-level entry-point."); + const componentDirectory = await buildComponentDirectory( + ctx, + path.resolve(args.path), + ); + + // No attempt to resolve args.path is made for entry points so they + // must be relative or absolute file paths, not npm packages. + // Whether we're bundling or discovering, we're done. + if (components.get(args.path)) { + // We always invoke esbuild in a try/catch. + // eslint-disable-next-line no-restricted-syntax + throw new Error( + `Entry point component "${args.path}" already registered.`, + ); } - let resolvedPath = undefined; - for (const candidate of candidates) { - try { - // --experimental-import-meta-resolve is required for - // `import.meta.resolve` so we'll use `require.resolve` - // until then. Hopefully they aren't too different. - const require = createRequire(args.resolveDir); - - // Sweet, this does all the Node.js stuff for us! - resolvedPath = require.resolve(candidate, { - paths: [args.resolveDir], - }); - break; - } catch (e: any) { - if (e.code === "MODULE_NOT_FOUND") { - continue; - } - // We always catch outside of an esbuild invocation. - // eslint-disable-next-line no-restricted-syntax - throw e; + components.set(args.path, componentDirectory); + return; + } + + const candidates = [args.path]; + const ext = path.extname(args.path); + if (ext === ".js") { + candidates.push(args.path.slice(0, -".js".length) + ".ts"); + } + if (ext !== ".js" && ext !== ".ts") { + candidates.push(args.path + ".js"); + candidates.push(args.path + ".ts"); + } + let resolvedPath = undefined; + for (const candidate of candidates) { + try { + // --experimental-import-meta-resolve is required for + // `import.meta.resolve` so we'll use `require.resolve` + // until then. Hopefully they aren't too different. + const require = createRequire(args.resolveDir); + resolvedPath = require.resolve(candidate, { + paths: [args.resolveDir], + }); + break; + } catch (e: any) { + if (e.code === "MODULE_NOT_FOUND") { + continue; } + // We always invoke esbuild in a try/catch. + // eslint-disable-next-line no-restricted-syntax + throw e; } - if (resolvedPath === undefined) { - // Let `esbuild` handle this itself. - verbose && logMessage(ctx, ` -> ${args.path} not found.`); + } + if (resolvedPath === undefined) { + verbose && logMessage(ctx, ` -> ${args.path} not found.`); + return; + } + + const parentDir = path.dirname(resolvedPath); + let imported = components.get(resolvedPath); + if (!imported) { + const isComponent = isComponentDirectory(ctx, parentDir, false); + if (isComponent.kind !== "ok") { + verbose && logMessage(ctx, " -> Not a component:", isComponent); return; } - - const parentDir = path.dirname(resolvedPath); - - let imported = components.get(resolvedPath); - if (!imported) { - const isComponent = isComponentDirectory(ctx, parentDir, false); - if (isComponent.kind !== "ok") { - verbose && logMessage(ctx, " -> Not a component:", isComponent); - return; - } - imported = isComponent.component; - components.set(resolvedPath, imported); - } - - verbose && - logMessage(ctx, " -> Component import! Recording it.", args.path); - - if (mode === "discover") { - return { - path: resolvedPath, - }; - } else { - // In bundle mode, transform external imports to use componentPaths: - // import rateLimiter from "convex_ratelimiter"; - // => import rateLimiter from `_componentDeps/${base64('../node_modules/convex_ratelimiter')}`; - - // A componentPath is path from the root component to the directory - // of the this component's definition file. + imported = isComponent.component; + components.set(resolvedPath, imported); + } + + verbose && + logMessage(ctx, " -> Component import! Recording it.", args.path); + + if (mode === "discover") { + return { + path: resolvedPath, + }; + } else { + // In bundle mode, transform external imports to use componentPaths: + // import rateLimiter from "convex_ratelimiter"; + // => import rateLimiter from `_componentDeps/${base64('../node_modules/convex_ratelimiter')}`; + + // A componentPath is path from the root component to the directory + // of the this component's definition file. + const componentPath = toComponentPath( + rootComponentDirectory, + imported, + ); + const encodedPath = hackyMapping(componentPath); + return { + path: encodedPath, + external: true, + }; + } + }); + // A more efficient version of this is possible as long as we control + // the bundling of the convex library: we can mark a path there external + // and generate the synthetic "./current-convex-component-path" module + // here. It changes our package more though so let's start with this. + if (mode === "bundle") { + build.onLoad( + { filter: /.*(component.config|app.config).*/ }, + async (args) => { + let text = ctx.fs.readUtf8File(args.path); + + const componentDirectory = await buildComponentDirectory( + ctx, + path.resolve(args.path), + ); const componentPath = toComponentPath( rootComponentDirectory, - imported, + componentDirectory, ); - const encodedPath = hackyMapping(componentPath); + text = text + .replace( + /defineComponent\(/g, + `defineComponent(${JSON.stringify(componentPath)},`, + ) + .replace( + /defineApp\(/g, + `defineApp(${JSON.stringify(componentPath)},`, + ); + return { - path: encodedPath, - external: true, + contents: text, + // TODO does this need to preserve original loader? + loader: "tsx", }; - } - }, - ); + }, + ); + } }, }; } /** The path on the deployment that identifier a component definition. */ -function hackyMapping(componentPath: string): string { +function hackyMapping(componentPath: ComponentPath): string { return `./_componentDeps/${Buffer.from(componentPath).toString("base64").replace(/=+$/, "")}`; } -// Use the metafile to discover the dependency graph in which component +// Share configuration between the component definition discovery and bundling passes. +const SHARED_ESBUILD_OPTIONS = { + bundle: true, + platform: "browser", + format: "esm", + target: "esnext", + + // false is the default for splitting. + // It's simpler to evaluate these on the server when we don't need a whole + // filesystem. Enabled this for speed once the server supports it. + splitting: false, + + // place output files in memory at their source locations + write: false, + outdir: "/", + outbase: "/", + + minify: true, + keepNames: true, + + metafile: true, +} as const satisfies BuildOptions; + +// Use the esbuild metafile to discover the dependency graph in which component // definitions are nodes. -// If it's cyclic, throw! That's no good! export async function componentGraph( ctx: Context, absWorkingDir: string, @@ -188,9 +232,8 @@ export async function componentGraph( }> { let result; try { - // The discover plugin collects component directories in .components. result = await esbuild.build({ - absWorkingDir, // mostly used for formatting error messages + absWorkingDir, // This is mostly useful for formatting error messages. entryPoints: [qualifiedDefinitionPath(rootComponentDirectory)], plugins: [ componentPlugin({ @@ -200,22 +243,10 @@ export async function componentGraph( rootComponentDirectory, }), ], - bundle: true, - platform: "browser", - format: "esm", - target: "esnext", - sourcemap: "external", sourcesContent: false, - // place output files in memory at their source locations - write: false, - outdir: "/", - outbase: "/", - - minify: true, - keepNames: true, - metafile: true, + ...SHARED_ESBUILD_OPTIONS, }); await registerEsbuildReads(ctx, absWorkingDir, result.metafile); } catch (err: any) { @@ -229,7 +260,6 @@ export async function componentGraph( } return await ctx.crash(1, "invalid filesystem data"); } - // TODO we're going to end up printing these warnings twice right now for (const warning of result.warnings) { console.log(chalk.yellow(`esbuild warning: ${warning.text}`)); } @@ -256,8 +286,9 @@ export function getDeps( /** * The returned dependency graph is an array of tuples of [importer, imported] * - * This doesn't work on just any esbuild metafile; it has to be run with - * the component esbuilt plugin run in "discover" mode so that it won't modify anything. + * This doesn't work on just any esbuild metafile because it assumes input + * imports have not been transformed. We run it on the metafile produced by + * the esbuild invocation that uses the component plugin in "discover" mode. */ async function findComponentDependencies( ctx: Context, @@ -266,13 +297,9 @@ async function findComponentDependencies( components: Map; dependencyGraph: [ComponentDirectory, ComponentDirectory][]; }> { - // The esbuild metafile has inputs as relative paths from cwd - // but the imports for each input are absolute paths or unqualified - // paths when marked external. const { inputs } = metafile; - // TODO compress these inputs so that everything that is not a component.config.ts - // or app.config.ts has its dependencies compacted down. - // Until then we just let these missing links slip through. + // This filter means we only supports *direct imports* of component definitions + // from other component definitions. const componentInputs = Object.keys(inputs).filter((path) => path.includes(".config."), ); @@ -312,14 +339,12 @@ async function findComponentDependencies( return { components, dependencyGraph }; } -/** - * Bundle definitions listed in directories. An app.config.ts must exist - * in the absWorkingDir. - * If a directory linked to is not listed then there will be external links - * with no corresponding definition bundle. - * That could be made to throw an error but maybe those are already available - * on the Convex definition filesystem somehow, e.g. builtin components. - */ +// NB: If a directory linked to is not a member of the passed +// componentDirectories array then there will be external links +// with no corresponding definition bundle. +// That could be made to throw an error but maybe those are already available +// on the Convex definition filesystem somehow, e.g. builtin components. +/** Bundle the component definitions listed. */ export async function bundleDefinitions( ctx: Context, absWorkingDir: string, @@ -333,9 +358,8 @@ export async function bundleDefinitions( }> { let result; try { - // TODO These should all come from ../../../bundler/index.js, at least helpers. result = await esbuild.build({ - absWorkingDir: absWorkingDir, + absWorkingDir, entryPoints: componentDirectories.map((dir) => qualifiedDefinitionPath(dir), ), @@ -347,40 +371,14 @@ export async function bundleDefinitions( rootComponentDirectory, }), ], - bundle: true, - platform: "browser", - format: "esm", - target: "esnext", - - // false is the default for splitting. - // It's simpler to evaluate these on the server when we don't need a whole - // filesystem. Enabled this for speed once the server supports it. - splitting: false, - - // whatever - sourcemap: false, - sourcesContent: false, - - // place output files in memory at their source locations - write: false, - outdir: "/", - outbase: "/", - - // debugging over bundle size for now, change later - minify: false, - keepNames: true, - - // Either we trust dependencies passed in or we build our own. - // Better to be consistent here? - metafile: true, + sourcemap: false, // we're just building a deps map + ...SHARED_ESBUILD_OPTIONS, }); await registerEsbuildReads(ctx, absWorkingDir, result.metafile); } catch (err: any) { logError(ctx, `esbuild failed: ${err}`); return await ctx.crash(1, "invalid filesystem data"); } - // TODO In theory we should get exactly the same errors here as we did the first time. Do something about that. - // TODO abstract out this esbuild wrapper stuff now that it's in two places if (result.errors.length) { for (const error of result.errors) { console.log(chalk.red(`esbuild error: ${error.text}`)); @@ -422,9 +420,14 @@ export async function bundleDefinitions( }); } - const [appBundle] = outputs.filter( + const appBundles = outputs.filter( (out) => out.directory.path === rootComponentDirectory.path, ); + if (appBundles.length !== 1) { + logError(ctx, "found wrong number of app bundles"); + return await ctx.crash(1, "fatal"); + } + const appBundle = appBundles[0]; const componentBundles = outputs.filter( (out) => out.directory.path !== rootComponentDirectory.path, ); @@ -478,7 +481,7 @@ export async function bundleImplementations( componentImplementations: { schema: Bundle; functions: Bundle[]; - definitionPath: string; + definitionPath: ComponentPath; }[]; }> { let appImplementation; @@ -492,8 +495,6 @@ export async function bundleImplementations( ); const schema = (await bundleSchema(ctx, resolvedPath))[0] || null; - // TODO figure out how this logic applies to non convex directories - // for components not defined in one. const entryPoints = await entryPointsByEnvironment( ctx, resolvedPath, @@ -506,17 +507,10 @@ export async function bundleImplementations( } = await bundle(ctx, resolvedPath, entryPoints.isolate, true, "browser"); if (convexResult.externalDependencies.size !== 0) { - logError(ctx, "TODO external deps"); + logError(ctx, "external dependencies not supported"); return await ctx.crash(1, "fatal"); } - // TODO Node compilation (how will Lambdas even work?) - const _nodeResult = { - bundles: [], - externalDependencies: new Map(), - bundledModuleNames: new Set(), - }; - const functions = convexResult.modules; if (isRoot) { appImplementation = { @@ -535,7 +529,6 @@ export async function bundleImplementations( } if (!appImplementation) { - // TODO should be enforced earlier logError(ctx, "No app implementation found"); return await ctx.crash(1, "fatal"); } @@ -543,16 +536,18 @@ export async function bundleImplementations( return { appImplementation, componentImplementations }; } -// TODO ensure this isn't broken with changes to workingdir location async function registerEsbuildReads( ctx: Context, absWorkingDir: string, metafile: Metafile, ) { for (const [relPath, input] of Object.entries(metafile.inputs)) { - // TODO: esbuild outputs paths prefixed with "(disabled)"" when bundling our internal - // udf-system package. The files do actually exist locally, though. if ( + // We rewrite these files so this integrity check isn't useful. + path.basename(relPath).includes("app.config") || + path.basename(relPath).includes("component.config") || + // TODO: esbuild outputs paths prefixed with "(disabled)" when bundling our internal + // udf-system package. The files do actually exist locally, though. relPath.indexOf("(disabled):") !== -1 || relPath.startsWith("wasm-binary:") || relPath.startsWith("wasm-stub:") diff --git a/src/cli/lib/components/definition/directoryStructure.ts b/src/cli/lib/components/definition/directoryStructure.ts index 124704a..19f0183 100644 --- a/src/cli/lib/components/definition/directoryStructure.ts +++ b/src/cli/lib/components/definition/directoryStructure.ts @@ -3,31 +3,26 @@ import { Context, logError } from "../../../../bundler/context.js"; import { DEFINITION_FILENAME, ROOT_DEFINITION_FILENAME } from "../constants.js"; /** - * Absolute paths to a component on the local filesystem. + * A component definition's location on the local filesystem, + * using absolute paths. * - * For module resolution it's useful to avoid resolving any symlinks: - * node modules may have different locations on disk, but should be understood - * to exist at the location + * For module resolution it may be useful to avoid resolving any symlinks: + * node modules are often symlinked by e.g. pnpm but relative paths should generally be + * understood from their symlink location. * - * ComponentDirectory *could* store the unqualifed import string used to find it. + * None of these properties are the import string, which might have been an unqualifed import * (e.g. 'convex-waitlist' instead of '../node_modules/convex-waitlist/component.config.ts') - * but it doesn't. */ export type ComponentDirectory = { name: string; path: string; definitionPath: string; }; -// If you want an abspath don't use these things! -// Goals: -// 1. no absolute paths should be sent to a Convex deployments -// 2. convex/app.config.js is not a hardcoded locations, since functionsDir changes -// when functionsDir changes, ideally /** * Qualify (ensure a leading dot) a path and make it relative to a working dir. - * Qualifying a path clarifies that it represents a local file system path, not - * a remote path on the npm registry. + * Qualifying a path clarifies to esbuild that it represents a local file system + * path, not a remote path on the npm registry. * * Because the path is made relative without resolving symlinks this is a reasonable * identifier for the component directory (given a consistent working directory). @@ -45,25 +40,25 @@ export function qualifiedDefinitionPath( } } -// The process cwd will be used to resolve a componentPath specified in the constructor. +// NB: The process cwd will be used to resolve the directory specified in the constructor. export function isComponentDirectory( ctx: Context, - componentPath: string, + directory: string, isRoot: boolean, ): | { kind: "ok"; component: ComponentDirectory } | { kind: "err"; why: string } { - if (!ctx.fs.exists(componentPath)) { + if (!ctx.fs.exists(directory)) { return { kind: "err", why: `Directory doesn't exist` }; } - const dirStat = ctx.fs.stat(componentPath); + const dirStat = ctx.fs.stat(directory); if (!dirStat.isDirectory()) { return { kind: "err", why: `Not a directory` }; } // Check that we have a definition file. const filename = isRoot ? ROOT_DEFINITION_FILENAME : DEFINITION_FILENAME; - const definitionPath = path.resolve(path.join(componentPath, filename)); + const definitionPath = path.resolve(path.join(directory, filename)); if (!ctx.fs.exists(definitionPath)) { return { kind: "err", @@ -80,8 +75,8 @@ export function isComponentDirectory( return { kind: "ok", component: { - name: isRoot ? "App" : path.basename(componentPath), - path: path.resolve(componentPath), + name: isRoot ? "App" : path.basename(directory), + path: path.resolve(directory), definitionPath: definitionPath, }, }; @@ -107,21 +102,23 @@ export async function buildComponentDirectory( return isComponent.component; } -// A component path is a useful concept on the server. -// Absolute paths should not reach the server because deploying -// with a repo checked out to a different location should not result -// in changes. -// Paths relative to the root of a project should not reach the server -// for similar but more theoritical reasons: a given convex functions -// directory should configure a deployment regardless of the package.json -// being used to do the deploy. -export type ComponentPath = string; +/** + * ComponentPath is the type of path sent to the server to identify a + * component definition. It is the unqualified (it never starts with "./") + * relative path from the convex directory of the app (root component) + * to the directory where a component definition lives. + * + * Note the convex/ directory of the root component is not necessarily + * the working directory. It is currently never the same as the working + * directory since `npx convex` must be invoked from the package root instead. + */ +export type ComponentPath = string & { __brand: "ComponentPath" }; export function toComponentPath( rootComponent: ComponentDirectory, component: ComponentDirectory, -) { - return path.relative(rootComponent.path, component.path); +): ComponentPath { + return path.relative(rootComponent.path, component.path) as ComponentPath; } export function toAbsolutePath( diff --git a/src/cli/lib/config.ts b/src/cli/lib/config.ts index c07b36a..c384519 100644 --- a/src/cli/lib/config.ts +++ b/src/cli/lib/config.ts @@ -612,6 +612,66 @@ interface BundledModuleInfo { platform: "node" | "convex"; } +/** A component definition spec contains enough information to create bundles + * of code that must be analyzed in order to construct a ComponentDefinition. + * + * Most paths are relative to the directory of the definitionPath. + */ +export type ComponentDefinitionSpec = { + /** This path is relative to the app (root component) directory. */ + definitionPath: string; + /** Dependencies are paths to the directory of the dependency component definition from the app (root component) directory */ + dependencies: string[]; + + // All other paths are relative to the directory of the definitionPath above. + + definition: Bundle; + schema: Bundle; + functions: Bundle[]; +}; + +export type AppDefinitionSpec = Omit< + ComponentDefinitionSpec, + "definitionPath" +> & { + // Only app (root) component specs contain an auth bundle. + auth: Bundle | null; +}; + +export type ComponentDefinitionSpecWithoutImpls = Omit< + ComponentDefinitionSpec, + "schema" | "functions" +>; +export type AppDefinitionSpecWithoutImpls = Omit< + AppDefinitionSpec, + "schema" | "functions" | "auth" +>; + +// TODO repetitive now, but this can do some denormalization if helpful +export function config2JSON( + adminKey: string, + functions: string, + udfServerVersion: string, + appDefinition: AppDefinitionSpec, + componentDefinitions: ComponentDefinitionSpec[], +): { + adminKey: string; + functions: string; + udfServerVersion: string; + appDefinition: AppDefinitionSpec; + componentDefinitions: ComponentDefinitionSpec[]; + nodeDependencies: []; +} { + return { + adminKey, + functions, + udfServerVersion, + appDefinition, + componentDefinitions, + nodeDependencies: [], + }; +} + export function configJSON( config: Config, adminKey: string, @@ -647,6 +707,81 @@ export type PushMetrics = { totalBeforePush: number; }; +type PushConfig2Response = { + externalDepsId: null | unknown; // this is a guess + appPackage: string; // like '9e0fbcbe-b2bc-40a3-9273-6a24896ba8ec', + componentPackages: Record /* like { + '../../convex_ratelimiter/ratelimiter': '4dab8e49-6e40-47fb-ae5b-f53f58ccd244', + '../examples/waitlist': 'b2eaba58-d320-4b84-85f1-476af834c17f' + },*/; + appAuth: unknown[]; + analysis: Record< + string, + { + definition: { + path: string; // same as key? + definitionType: { type: "app" } | unknown; + childComponents: unknown[]; + exports: unknown; + }; + schema: { tables: unknown[]; schemaValidation: boolean }; + // really this is "modules" + functions: Record< + string, + { + functions: unknown[]; + httpRoutes: null | unknown; + cronSpecs: null | unknown; + sourceMapped: unknown; + } + >; + } + >; +}; + +/** Push configuration2 to the given remote origin. */ +export async function pushConfig2( + ctx: Context, + adminKey: string, + url: string, + functions: string, + udfServerVersion: string, + appDefinition: AppDefinitionSpec, + componentDefinitions: ComponentDefinitionSpec[], +): Promise { + const serializedConfig = config2JSON( + adminKey, + functions, + udfServerVersion, + appDefinition, + componentDefinitions, + ); + /* + const custom = (_k: string | number, s: any) => + typeof s === "string" + ? s.slice(0, 80) + (s.length > 80 ? "..............." : "") + : s; + console.log(JSON.stringify(serializedConfig, custom, 2)); + */ + const fetch = deploymentFetch(url); + changeSpinner(ctx, "Analyzing and deploying source code..."); + try { + const response = await fetch("/api/deploy2/start_push", { + body: JSON.stringify(serializedConfig), + method: "POST", + headers: { + "Content-Type": "application/json", + "Convex-Client": `npm-cli-${version}`, + }, + }); + return await response.json(); + } catch (error: unknown) { + // TODO incorporate AuthConfigMissingEnvironmentVariable logic + logFailure(ctx, "Error: Unable to start push to " + url); + return await logAndHandleFetchError(ctx, error); + } +} + /** Push configuration to the given remote origin. */ export async function pushConfig( ctx: Context, diff --git a/src/server/components/definition.ts b/src/server/components/definition.ts new file mode 100644 index 0000000..773d301 --- /dev/null +++ b/src/server/components/definition.ts @@ -0,0 +1,49 @@ +// These reflect server types. +export type ComponentDefinitionExport = { + name: string; + // how will we figure this out? + path: string; + definitionType: { + type: "childComponent"; + name: string; + args: [string, { type: "value"; value: string }][]; + }; + childComponents: []; + exports: { type: "branch"; branch: [] }; +}; + +// These reflect server types. +// type ComponentDefinitionType +export type ComponentDefinitionType = { + type: "childComponent"; + name: string; + args: [string, { type: "value"; value: string }][]; +}; +export type AppDefinitionType = { type: "app" }; + +type ComponentInstantiation = { + name: string; + // This is a ComponentPath. + path: string; + args: [string, { type: "value"; value: string }][]; +}; + +type ComponentExport = + | { type: "branch"; branch: ComponentExport[] } + | { type: "leaf"; leaf: string }; + +// The type expected from the internal .export() +// method of a component or app definition. +export type ComponentDefinitionAnalysis = { + name: string; + path: string; + definitionType: ComponentDefinitionType; + childComponents: ComponentInstantiation[]; + exports: ComponentExport; +}; +export type AppDefinitionAnalysis = { + path: string; + definitionType: AppDefinitionType; + childComponents: ComponentInstantiation[]; + exports: ComponentExport; +}; diff --git a/src/server/components/index.ts b/src/server/components/index.ts new file mode 100644 index 0000000..480d112 --- /dev/null +++ b/src/server/components/index.ts @@ -0,0 +1,254 @@ +import { + Infer, + ObjectType, + PropertyValidators, + convexToJson, +} from "../../values/index.js"; +import { + AppDefinitionAnalysis, + ComponentDefinitionAnalysis, + ComponentDefinitionType, +} from "./definition"; + +type ComponentArgsMethod< + IsRoot extends boolean, + Args extends PropertyValidators, +> = IsRoot extends false + ? { + args( + args: AdditionalArgs, + ): ComponentDefinition; + } + : // eslint-disable-next-line @typescript-eslint/ban-types + {}; + +type CommonComponentsDefinition< + IsRoot extends boolean, + Args extends PropertyValidators, +> = { + install>( + name: string, + definition: Definition, + args: ObjectType>, + ): ComponentDefinition; +}; + +type ComponentDefinition< + IsRoot extends boolean, + Args extends PropertyValidators, +> = CommonComponentsDefinition & + ComponentArgsMethod; + +type CommonDefinitionData = { + _isRoot: boolean; + _childComponents: [string, ImportedComonentDefinition, Record][]; +}; +type ComponentDefinitionData = CommonDefinitionData & { + _args: PropertyValidators; + _name: string; +}; +type AppDefinitionData = CommonDefinitionData; + +type ExtractArgs = T extends ComponentDefinition ? P : never; + +type DefineComponent = ( + name: string, + // eslint-disable-next-line @typescript-eslint/ban-types +) => ComponentDefinition; + +type DefineApp = () => ComponentDefinition< + true, + Args +>; + +function componentArgsBuilder( + this: ComponentDefinition & ComponentDefinitionData, + additionalArgs: PropertyValidators, +): ComponentDefinition & ComponentDefinitionData { + return { ...this, _args: { ...this._args, ...additionalArgs } }; +} + +function installBuilder>( + this: ComponentDefinition & ComponentDefinitionData, + name: string, + definition: Definition, + args: Infer>, +) { + const importedComponentDefinition = + definition as unknown as ImportedComonentDefinition; + return { + ...this, + _childComponents: [ + ...this._childComponents, + [name, importedComponentDefinition, args], + ], + }; +} + +// Injected by the bundler +type BundlerAssignedComponentData = { + _componentPath: string; +}; + +// At runtime when you import a ComponentDefinition, this is all it is +type ImportedComonentDefinition = { + componentDefinitionPath: string; +}; + +function exportAppForAnalysis( + this: ComponentDefinition & + ComponentDefinitionData & + BundlerAssignedComponentData, +): AppDefinitionAnalysis { + if (!this._isRoot) { + throw new Error( + "`exportComponentForAnalysis()` must run on a root component.", + ); + } + const componentPath = this._componentPath; + if (!componentPath === undefined) { + throw new Error( + `ComponentPath not found in ${JSON.stringify(this, null, 2)}`, + ); + } + const definitionType = { type: "app" as const }; + const childComponents = serializeChildComponents(this._childComponents); + + return { + // this is a component path. It will need to be provided by the bundler somehow. + // An esbuild plugin needs to take over to do this during bundling. + path: componentPath, + definitionType, + childComponents: childComponents as any, + exports: { type: "branch", branch: [] }, + }; +} + +function serializeChildComponents( + childComponents: [string, ImportedComonentDefinition, Record][], +): { + name: string; + path: string; + args: [string, { type: "value"; value: string }][]; +}[] { + return childComponents.map(([name, definition, p]) => { + const args: [string, { type: "value"; value: string }][] = []; + for (const [name, value] of Object.entries(p)) { + args.push([ + name, + { type: "value", value: JSON.stringify(convexToJson(value)) }, + ]); + } + // we know that components carry this extra information + const path = definition.componentDefinitionPath; + if (!path) + throw new Error( + "no .componentPath for component definition " + + JSON.stringify(definition, null, 2), + ); + + return { + name: name!, + path: path!, + args, + }; + }); +} + +function exportComponentForAnalysis( + this: ComponentDefinition & + ComponentDefinitionData & + BundlerAssignedComponentData, +): ComponentDefinitionAnalysis { + if (this._isRoot) { + throw new Error( + "`exportComponentForAnalysis()` cannot run on a root component.", + ); + } + const componentPath = this._componentPath; + if (!componentPath === undefined) { + throw new Error( + `ComponentPath not found in ${JSON.stringify(this, null, 2)}`, + ); + } + const args: [string, { type: "value"; value: string }][] = Object.entries( + this._args, + ).map(([name, validator]) => [ + name, + { + type: "value", + value: JSON.stringify(validator.json), + }, + ]); + const definitionType: ComponentDefinitionType = { + type: "childComponent" as const, + name: this._name, + args, + }; + const childComponents = serializeChildComponents(this._childComponents); + + return { + name: this._name, + // this is a component path. It will need to be provided by the bundler somehow. + // An esbuild plugin needs to take over to do this during bundling. + path: componentPath, + definitionType, + childComponents: childComponents as any, + exports: { type: "branch", branch: [] }, + }; +} + +function defineComponentImpl( + componentPath: string, // secret first argument inserted by the bundler + name: string, + // eslint-disable-next-line @typescript-eslint/ban-types +): ComponentDefinition & + ComponentDefinitionData & { + export: () => ComponentDefinitionAnalysis; + } & BundlerAssignedComponentData { + if (name === undefined) { + throw new Error( + "defineComponentImpl needs its secret first argument filled in by the bundler", + ); + } + return { + _isRoot: false, + _name: name, + _args: {}, + _childComponents: [], + _componentPath: componentPath, + args: componentArgsBuilder, + export: exportComponentForAnalysis, + install: installBuilder, + }; +} + +function defineAppImpl( + componentPath: string, // secret first argument inserted by the bundler +): ComponentDefinition & + AppDefinitionData & { + export: () => AppDefinitionAnalysis; + } & BundlerAssignedComponentData { + if (componentPath === undefined) { + throw new Error( + "defineComponentImpl needs its secret first argument filled in by the bundler", + ); + } + return { + _isRoot: true, + _childComponents: [], + _componentPath: componentPath, + export: exportAppForAnalysis, + install: installBuilder, + }; +} + +/** + * @internal + */ +export const defineComponent = + defineComponentImpl as unknown as DefineComponent; +/** + * @internal + */ +export const defineApp = defineAppImpl as unknown as DefineApp; diff --git a/src/server/index.ts b/src/server/index.ts index 9b249a1..6508171 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -156,6 +156,11 @@ export type { FunctionReturnType, } from "./api.js"; +/** + * @internal + */ +export { defineApp, defineComponent } from "./components/index.js"; + /** * @internal */