diff --git a/packages/gatsby/src/bootstrap/index.ts b/packages/gatsby/src/bootstrap/index.ts index 0bb37c8b7ab9c..5e8ec23d599d7 100644 --- a/packages/gatsby/src/bootstrap/index.ts +++ b/packages/gatsby/src/bootstrap/index.ts @@ -33,7 +33,6 @@ export async function bootstrap( const bootstrapContext: IBuildContext = { ...initialContext, parentSpan, - firstRun: true, } const context = { diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 3d02f356b6f2c..80e2e0c83b9b5 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -4,51 +4,20 @@ import chalk from "chalk" import telemetry from "gatsby-telemetry" import express from "express" import inspector from "inspector" -import { bootstrapSchemaHotReloader } from "../bootstrap/schema-hot-reloader" -import bootstrapPageHotReloader from "../bootstrap/page-hot-reloader" import { initTracer } from "../utils/tracer" -import db from "../db" import { detectPortInUseAndPrompt } from "../utils/detect-port-in-use-and-prompt" import onExit from "signal-exit" -import queryUtil from "../query" -import queryWatcher from "../query/query-watcher" -import * as requiresWriter from "../bootstrap/requires-writer" -import { waitUntilAllJobsComplete } from "../utils/wait-until-jobs-complete" import { userPassesFeedbackRequestHeuristic, showFeedbackRequest, } from "../utils/feedback" -import { startRedirectListener } from "../bootstrap/redirects-writer" import { markWebpackStatusAsPending } from "../utils/webpack-status" import { IProgram, IDebugInfo } from "./types" -import { - startWebpackServer, - writeOutRequires, - IBuildContext, - initialize, - postBootstrap, - rebuildSchemaWithSitePage, - writeOutRedirects, -} from "../services" -import { boundActionCreators } from "../redux/actions" -import { ProgramStatus } from "../redux/types" -import { - MachineConfig, - AnyEventObject, - assign, - Machine, - DoneEventObject, - interpret, - Actor, - Interpreter, - State, -} from "xstate" -import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer" -import { IDataLayerContext } from "../state-machines/data-layer/types" +import { interpret } from "xstate" import { globalTracer } from "opentracing" -import { IQueryRunningContext } from "../state-machines/query-running/types" -import { queryRunningMachine } from "../state-machines/query-running" +import { developMachine } from "../state-machines/develop" +import { logTransitions } from "../utils/state-machine-logging" const tracer = globalTracer() @@ -100,12 +69,14 @@ const openDebuggerPort = (debugInfo: IDebugInfo): void => { } module.exports = async (program: IDevelopArgs): Promise => { + if (program.verbose) { + reporter.setVerbose(true) + } + if (program.debugInfo) { openDebuggerPort(program.debugInfo) } - const bootstrapSpan = tracer.startSpan(`bootstrap`) - // We want to prompt the feedback request when users quit develop // assuming they pass the heuristic check to know they are a user // we want to request feedback from, and we're not annoying them. @@ -148,178 +119,19 @@ module.exports = async (program: IDevelopArgs): Promise => { } const app = express() + const parentSpan = tracer.startSpan(`bootstrap`) - const developConfig: MachineConfig = { - id: `build`, - initial: `initializing`, - states: { - initializing: { - invoke: { - src: `initialize`, - onDone: { - target: `initializingDataLayer`, - actions: `assignStoreAndWorkerPool`, - }, - }, - }, - initializingDataLayer: { - invoke: { - src: `initializeDataLayer`, - data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { - return { parentSpan, store, firstRun: true } - }, - onDone: { - actions: `assignDataLayer`, - target: `finishingBootstrap`, - }, - }, - }, - finishingBootstrap: { - invoke: { - src: async ({ - gatsbyNodeGraphQLFunction, - }: IBuildContext): Promise => { - // These were previously in `bootstrap()` but are now - // in part of the state machine that hasn't been added yet - await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan }) - - await writeOutRedirects({ parentSpan: bootstrapSpan }) - - startRedirectListener() - bootstrapSpan.finish() - await postBootstrap({ parentSpan: bootstrapSpan }) - - // These are the parts that weren't in bootstrap - - // Start the createPages hot reloader. - bootstrapPageHotReloader(gatsbyNodeGraphQLFunction) - - // Start the schema hot reloader. - bootstrapSchemaHotReloader() - }, - onDone: { - target: `runningQueries`, - }, - }, - }, - runningQueries: { - invoke: { - src: `runQueries`, - data: ({ - program, - store, - parentSpan, - gatsbyNodeGraphQLFunction, - graphqlRunner, - firstRun, - }: IBuildContext): IQueryRunningContext => { - return { - firstRun, - program, - store, - parentSpan, - gatsbyNodeGraphQLFunction, - graphqlRunner, - } - }, - onDone: { - target: `doingEverythingElse`, - }, - }, - }, - doingEverythingElse: { - invoke: { - src: async ({ workerPool, store, app }): Promise => { - // All the stuff that's not in the state machine yet - - await writeOutRequires({ store }) - boundActionCreators.setProgramStatus( - ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED - ) - - await db.saveState() + const machine = developMachine.withContext({ + program, + parentSpan, + app, + }) - await waitUntilAllJobsComplete() - requiresWriter.startListener() - db.startAutosave() - queryUtil.startListeningToDevelopQueue({ - graphqlTracing: program.graphqlTracing, - }) - queryWatcher.startWatchDeletePage() + const service = interpret(machine) - await startWebpackServer({ program, app, workerPool, store }) - }, - onDone: { - actions: assign({ firstRun: false }), - }, - }, - }, - }, + if (program.verbose) { + logTransitions(service) } - const service = interpret( - Machine(developConfig, { - services: { - initializeDataLayer: dataLayerMachine, - initialize, - runQueries: queryRunningMachine, - }, - actions: { - assignStoreAndWorkerPool: assign( - (_context, event) => { - const { store, workerPool } = event.data - return { - store, - workerPool, - } - } - ), - assignDataLayer: assign( - (_, { data }): DataLayerResult => data - ), - }, - }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true }) - ) - - const isInterpreter = ( - actor: Actor | Interpreter - ): actor is Interpreter => `machine` in actor - - const listeners = new WeakSet() - let last: State - - service.onTransition(state => { - if (!last) { - last = state - } else if (!state.changed || last.matches(state)) { - return - } - last = state - reporter.verbose(`Transition to ${JSON.stringify(state.value)}`) - // eslint-disable-next-line no-unused-expressions - service.children?.forEach(child => { - // We want to ensure we don't attach a listener to the same - // actor. We don't need to worry about detaching the listener - // because xstate handles that for us when the actor is stopped. - - if (isInterpreter(child) && !listeners.has(child)) { - let sublast = child.state - child.onTransition(substate => { - if (!sublast) { - sublast = substate - } else if (!substate.changed || sublast.matches(substate)) { - return - } - sublast = substate - reporter.verbose( - `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify( - substate.value - )}` - ) - }) - listeners.add(child) - } - }) - }) service.start() } diff --git a/packages/gatsby/src/commands/types.ts b/packages/gatsby/src/commands/types.ts index bc4fb90f8f54c..bcd4889c88395 100644 --- a/packages/gatsby/src/commands/types.ts +++ b/packages/gatsby/src/commands/types.ts @@ -31,6 +31,7 @@ export interface IProgram { inspect?: number inspectBrk?: number graphqlTracing?: boolean + verbose?: boolean setStore?: (store: Store) => void } diff --git a/packages/gatsby/src/query/index.js b/packages/gatsby/src/query/index.js index a8c9d7a96773b..2a699896d29c7 100644 --- a/packages/gatsby/src/query/index.js +++ b/packages/gatsby/src/query/index.js @@ -366,6 +366,7 @@ const enqueueExtractedPageComponent = componentPath => { module.exports = { calcInitialDirtyQueryIds, + calcDirtyQueryIds, processPageQueries, processStaticQueries, groupQueryIds, diff --git a/packages/gatsby/src/query/query-watcher.js b/packages/gatsby/src/query/query-watcher.js index 4937ad723888c..c434d42a0c534 100644 --- a/packages/gatsby/src/query/query-watcher.js +++ b/packages/gatsby/src/query/query-watcher.js @@ -8,7 +8,6 @@ * - Whenever a query changes, re-run all pages that rely on this query. ***/ -const _ = require(`lodash`) const chokidar = require(`chokidar`) const path = require(`path`) @@ -196,8 +195,7 @@ exports.extractQueries = ({ parentSpan } = {}) => { // During development start watching files to recompile & run // queries on the fly. - // TODO: move this into a spawned service, and emit events rather than - // directly triggering the compilation + // TODO: move this into a spawned service if (process.env.NODE_ENV !== `production`) { watch(store.getState().program.directory) } @@ -222,10 +220,6 @@ const watchComponent = componentPath => { } } -const debounceCompile = _.debounce(() => { - updateStateAndRunQueries() -}, 100) - const watch = async rootDir => { if (watcher) return @@ -238,13 +232,18 @@ const watch = async rootDir => { }) watcher = chokidar - .watch([ - slash(path.join(rootDir, `/src/**/*.{js,jsx,ts,tsx}`)), - ...packagePaths, - ]) + .watch( + [slash(path.join(rootDir, `/src/**/*.{js,jsx,ts,tsx}`)), ...packagePaths], + { ignoreInitial: true } + ) .on(`change`, path => { - report.pendingActivity({ id: `query-extraction` }) - debounceCompile() + emitter.emit(`QUERY_FILE_CHANGED`, path) + }) + .on(`add`, path => { + emitter.emit(`QUERY_FILE_CHANGED`, path) + }) + .on(`unlink`, path => { + emitter.emit(`QUERY_FILE_CHANGED`, path) }) filesToWatch.forEach(filePath => watcher.add(filePath)) diff --git a/packages/gatsby/src/services/calculate-dirty-queries.ts b/packages/gatsby/src/services/calculate-dirty-queries.ts index adf3bd7501bb5..4309074c0fab4 100644 --- a/packages/gatsby/src/services/calculate-dirty-queries.ts +++ b/packages/gatsby/src/services/calculate-dirty-queries.ts @@ -1,18 +1,24 @@ -import { calcInitialDirtyQueryIds, groupQueryIds } from "../query" +import { + calcInitialDirtyQueryIds, + calcDirtyQueryIds, + groupQueryIds, +} from "../query" import { IGroupedQueryIds } from "./" import { IQueryRunningContext } from "../state-machines/query-running/types" import { assertStore } from "../utils/assert-store" export async function calculateDirtyQueries({ store, + firstRun, }: Partial): Promise<{ queryIds: IGroupedQueryIds }> { assertStore(store) const state = store.getState() - // TODO: Check filesDirty from context - const queryIds = calcInitialDirtyQueryIds(state) + const queryIds = firstRun + ? calcInitialDirtyQueryIds(state) + : calcDirtyQueryIds(state) return { queryIds: groupQueryIds(queryIds) } } diff --git a/packages/gatsby/src/services/create-pages-statefully.ts b/packages/gatsby/src/services/create-pages-statefully.ts index a1c4798077521..163263006c9cb 100644 --- a/packages/gatsby/src/services/create-pages-statefully.ts +++ b/packages/gatsby/src/services/create-pages-statefully.ts @@ -5,6 +5,7 @@ import { IDataLayerContext } from "../state-machines/data-layer/types" export async function createPagesStatefully({ parentSpan, gatsbyNodeGraphQLFunction, + deferNodeMutation, }: Partial): Promise { // A variant on createPages for plugins that want to // have full control over adding/removing pages. The normal @@ -21,7 +22,7 @@ export async function createPagesStatefully({ traceId: `initial-createPagesStatefully`, waitForCascadingActions: true, parentSpan: activity.span, - // deferNodeMutation: true, //later + deferNodeMutation, }, { activity, diff --git a/packages/gatsby/src/services/create-pages.ts b/packages/gatsby/src/services/create-pages.ts index 0f456bf26914d..797da30ecce66 100644 --- a/packages/gatsby/src/services/create-pages.ts +++ b/packages/gatsby/src/services/create-pages.ts @@ -2,6 +2,8 @@ import reporter from "gatsby-cli/lib/reporter" import apiRunnerNode from "../utils/api-runner-node" import { IDataLayerContext } from "../state-machines/data-layer/types" import { assertStore } from "../utils/assert-store" +import { IGatsbyPage } from "../redux/types" +import { deleteUntouchedPages, findChangedPages } from "../utils/changed-pages" export async function createPages({ parentSpan, @@ -16,8 +18,8 @@ export async function createPages({ parentSpan, }) activity.start() - // const timestamp = Date.now() - // const currentPages = new Map(store.getState().pages) + const timestamp = Date.now() + const currentPages = new Map(store.getState().pages) await apiRunnerNode( `createPages`, @@ -38,31 +40,31 @@ export async function createPages({ ) activity.end() - // reporter.info(`Checking for deleted pages`) + reporter.verbose(`Checking for deleted pages`) - // const deletedPages = deleteUntouchedPages(store.getState().pages, timestamp) + const deletedPages = deleteUntouchedPages(store.getState().pages, timestamp) - // reporter.info( - // `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` - // ) + reporter.verbose( + `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` + ) - // const tim = reporter.activityTimer(`Checking for changed pages`) - // tim.start() + const tim = reporter.activityTimer(`Checking for changed pages`) + tim.start() - // const { changedPages } = findChangedPages( - // currentPages, - // store.getState().pages - // ) + const { changedPages } = findChangedPages( + currentPages, + store.getState().pages + ) - // reporter.info( - // `Found ${changedPages.length} changed page${ - // changedPages.length === 1 ? `` : `s` - // }` - // ) - // tim.end() + reporter.verbose( + `Found ${changedPages.length} changed page${ + changedPages.length === 1 ? `` : `s` + }` + ) + tim.end() return { - changedPages: [], - deletedPages: [], + changedPages, + deletedPages, } } diff --git a/packages/gatsby/src/services/index.ts b/packages/gatsby/src/services/index.ts index 3025520f9074b..7fc3a0c99f47c 100644 --- a/packages/gatsby/src/services/index.ts +++ b/packages/gatsby/src/services/index.ts @@ -18,6 +18,7 @@ import { runStaticQueries } from "./run-static-queries" import { runPageQueries } from "./run-page-queries" import { waitUntilAllJobsComplete } from "../utils/wait-until-jobs-complete" +import { runMutationBatch } from "./run-mutation-batch" export * from "./types" @@ -38,6 +39,7 @@ export { writeOutRedirects, startWebpackServer, rebuildSchemaWithSitePage, + runMutationBatch, } export const buildServices: Record> = { diff --git a/packages/gatsby/src/services/listen-for-mutations.ts b/packages/gatsby/src/services/listen-for-mutations.ts new file mode 100644 index 0000000000000..dc21cf75edb95 --- /dev/null +++ b/packages/gatsby/src/services/listen-for-mutations.ts @@ -0,0 +1,32 @@ +import { emitter } from "../redux" +import { InvokeCallback, Sender } from "xstate" + +export const listenForMutations: InvokeCallback = (callback: Sender) => { + const emitMutation = (event: unknown): void => { + callback({ type: `ADD_NODE_MUTATION`, payload: event }) + } + + const emitFileChange = (event: unknown): void => { + callback({ type: `SOURCE_FILE_CHANGED`, payload: event }) + } + + const emitQueryChange = (event: unknown): void => { + callback({ type: `QUERY_FILE_CHANGED`, payload: event }) + } + + const emitWebhook = (event: unknown): void => { + callback({ type: `WEBHOOK_RECEIVED`, payload: event }) + } + + emitter.on(`ENQUEUE_NODE_MUTATION`, emitMutation) + emitter.on(`WEBHOOK_RECEIVED`, emitWebhook) + emitter.on(`SOURCE_FILE_CHANGED`, emitFileChange) + emitter.on(`QUERY_FILE_CHANGED`, emitQueryChange) + + return function unsubscribeFromMutationListening(): void { + emitter.off(`ENQUEUE_NODE_MUTATION`, emitMutation) + emitter.off(`SOURCE_FILE_CHANGED`, emitFileChange) + emitter.off(`WEBHOOK_RECEIVED`, emitWebhook) + emitter.off(`QUERY_FILE_CHANGED`, emitQueryChange) + } +} diff --git a/packages/gatsby/src/services/run-mutation-batch.ts b/packages/gatsby/src/services/run-mutation-batch.ts new file mode 100644 index 0000000000000..51b36f481008a --- /dev/null +++ b/packages/gatsby/src/services/run-mutation-batch.ts @@ -0,0 +1,24 @@ +import { IMutationAction } from "../state-machines/data-layer/types" +import { Store, AnyAction } from "redux" +import { IGatsbyState } from "../redux/types" +import { IWaitingContext } from "../state-machines/waiting/types" +import { assertStore } from "../utils/assert-store" +import { actions } from "../redux/actions" + +const callRealApi = ( + event: IMutationAction, + store?: Store +): void => { + assertStore(store) + const { type, payload } = event + if (type in actions) { + store.dispatch(actions[type](...payload)) + } +} + +// Consume the entire batch and run actions +export const runMutationBatch = async ({ + runningBatch = [], + store, +}: Partial): Promise => + Promise.all(runningBatch.map(payload => callRealApi(payload, store))) diff --git a/packages/gatsby/src/services/source-nodes.ts b/packages/gatsby/src/services/source-nodes.ts index caf426815c64e..e14498d33537e 100644 --- a/packages/gatsby/src/services/source-nodes.ts +++ b/packages/gatsby/src/services/source-nodes.ts @@ -2,13 +2,14 @@ import sourceNodesAndRemoveStaleNodes from "../utils/source-nodes" import reporter from "gatsby-cli/lib/reporter" import { IDataLayerContext } from "../state-machines/data-layer/types" import { assertStore } from "../utils/assert-store" -// import { findChangedPages } from "../utils/check-for-changed-pages" -// import { IGatsbyPage } from "../redux/types" +import { IGatsbyPage } from "../redux/types" +import { findChangedPages } from "../utils/changed-pages" export async function sourceNodes({ parentSpan, webhookBody, store, + deferNodeMutation = false, }: Partial): Promise<{ deletedPages: string[] changedPages: string[] @@ -19,10 +20,10 @@ export async function sourceNodes({ parentSpan, }) activity.start() - // const currentPages = new Map(store.getState().pages) + const currentPages = new Map(store.getState().pages) await sourceNodesAndRemoveStaleNodes({ parentSpan: activity.span, - // deferNodeMutation: !!(webhookBody && Object.keys(webhookBody).length), // Coming soon + deferNodeMutation, webhookBody, }) @@ -34,31 +35,30 @@ export async function sourceNodes({ .join(`, `)}]` ) - // reporter.info(`Checking for deleted pages`) + reporter.verbose(`Checking for deleted pages`) - // Add this back when we enable page creation outside of onCreatePages - // const tim = reporter.activityTimer(`Checking for changed pages`) - // tim.start() + const tim = reporter.activityTimer(`Checking for changed pages`) + tim.start() - // const { changedPages, deletedPages } = findChangedPages( - // currentPages, - // store.getState().pages - // ) + const { changedPages, deletedPages } = findChangedPages( + currentPages, + store.getState().pages + ) - // reporter.info( - // `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` - // ) + reporter.verbose( + `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` + ) - // reporter.info( - // `Found ${changedPages.length} changed page${ - // changedPages.length === 1 ? `` : `s` - // }` - // ) - // tim.end() + reporter.verbose( + `Found ${changedPages.length} changed page${ + changedPages.length === 1 ? `` : `s` + }` + ) + tim.end() activity.end() return { - deletedPages: [], - changedPages: [], + deletedPages, + changedPages, } } diff --git a/packages/gatsby/src/services/start-webpack-server.ts b/packages/gatsby/src/services/start-webpack-server.ts index d05bb60190bad..1a6f6e474471f 100644 --- a/packages/gatsby/src/services/start-webpack-server.ts +++ b/packages/gatsby/src/services/start-webpack-server.ts @@ -13,7 +13,7 @@ import { import { printDeprecationWarnings } from "../utils/print-deprecation-warnings" import { printInstructions } from "../utils/print-instructions" import { prepareUrls } from "../utils/prepare-urls" -import { startServer } from "../utils/start-server" +import { startServer, IWebpackWatchingPauseResume } from "../utils/start-server" import { WebsocketManager } from "../utils/websocket-manager" import { IBuildContext } from "./" import { @@ -31,15 +31,17 @@ export async function startWebpackServer({ }: Partial): Promise<{ compiler: Compiler websocketManager: WebsocketManager + webpackWatching: IWebpackWatchingPauseResume }> { if (!program || !app || !store) { report.panic(`Missing required params`) } - let { compiler, webpackActivity, websocketManager } = await startServer( - program, - app, - workerPool - ) + let { + compiler, + webpackActivity, + websocketManager, + webpackWatching, + } = await startServer(program, app, workerPool) compiler.hooks.invalid.tap(`log compiling`, function () { if (!webpackActivity) { @@ -158,8 +160,7 @@ export async function startWebpackServer({ markWebpackStatusAsDone() done() - - resolve({ compiler, websocketManager }) + resolve({ compiler, websocketManager, webpackWatching }) }) }) } diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index 1b961b5479370..6a023df96242f 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -6,6 +6,10 @@ import { Store, AnyAction } from "redux" import { IGatsbyState } from "../redux/types" import { Express } from "express" import JestWorker from "jest-worker" +import { Actor, AnyEventObject } from "xstate" +import { Compiler } from "webpack" +import { WebsocketManager } from "../utils/websocket-manager" +import { IWebpackWatchingPauseResume } from "../utils/start-server" export interface IGroupedQueryIds { pageQueryIds: string[] staticQueryIds: string[] @@ -14,9 +18,9 @@ export interface IGroupedQueryIds { export interface IMutationAction { type: string payload: unknown[] + resolve?: (result: unknown) => void } export interface IBuildContext { - firstRun: boolean program?: IProgram store?: Store parentSpan?: Span @@ -27,4 +31,11 @@ export interface IBuildContext { refresh?: boolean workerPool?: JestWorker app?: Express + nodesMutatedDuringQueryRun?: boolean + mutationListener?: Actor + nodeMutationBatch?: IMutationAction[] + compiler?: Compiler + websocketManager?: WebsocketManager + webpackWatching?: IWebpackWatchingPauseResume + queryFilesDirty?: boolean } diff --git a/packages/gatsby/src/services/write-out-redirects.ts b/packages/gatsby/src/services/write-out-redirects.ts index 5745db6dc6fdf..18822eb781ff3 100644 --- a/packages/gatsby/src/services/write-out-redirects.ts +++ b/packages/gatsby/src/services/write-out-redirects.ts @@ -1,5 +1,8 @@ import reporter from "gatsby-cli/lib/reporter" -import { writeRedirects } from "../bootstrap/redirects-writer" +import { + writeRedirects, + startRedirectListener, +} from "../bootstrap/redirects-writer" import { IQueryRunningContext } from "../state-machines/query-running/types" export async function writeOutRedirects({ @@ -11,5 +14,6 @@ export async function writeOutRedirects({ }) activity.start() await writeRedirects() + startRedirectListener() activity.end() } diff --git a/packages/gatsby/src/state-machines/data-layer/actions.ts b/packages/gatsby/src/state-machines/data-layer/actions.ts index 72f89e5ca8ee5..0ae2747a536d4 100644 --- a/packages/gatsby/src/state-machines/data-layer/actions.ts +++ b/packages/gatsby/src/state-machines/data-layer/actions.ts @@ -1,24 +1,15 @@ -import { - assign, - AnyEventObject, - ActionFunction, - AssignAction, - DoneInvokeEvent, - ActionFunctionMap, -} from "xstate" +import { assign, DoneInvokeEvent, ActionFunctionMap } from "xstate" import { createGraphQLRunner } from "../../bootstrap/create-graphql-runner" import reporter from "gatsby-cli/lib/reporter" import { IDataLayerContext } from "./types" +import { callApi, markNodesDirty } from "../develop/actions" import { assertStore } from "../../utils/assert-store" +import { GraphQLRunner } from "../../query/graphql-runner" const concatUnique = (array1: T[] = [], array2: T[] = []): T[] => Array.from(new Set(array1.concat(array2))) -type BuildMachineAction = - | ActionFunction - | AssignAction - -export const assignChangedPages: BuildMachineAction = assign< +export const assignChangedPages = assign< IDataLayerContext, DoneInvokeEvent<{ changedPages: string[] @@ -31,19 +22,22 @@ export const assignChangedPages: BuildMachineAction = assign< } }) -export const assignGatsbyNodeGraphQL: BuildMachineAction = assign< - IDataLayerContext ->({ - gatsbyNodeGraphQLFunction: ({ store }: IDataLayerContext) => { +export const assignGraphQLRunners = assign( + ({ store, program }) => { assertStore(store) - return createGraphQLRunner(store, reporter) - }, -}) + return { + gatsbyNodeGraphQLFunction: createGraphQLRunner(store, reporter), + graphqlRunner: new GraphQLRunner(store, { + collectStats: true, + graphqlTracing: program?.graphqlTracing, + }), + } + } +) -export const dataLayerActions: ActionFunctionMap< - IDataLayerContext, - AnyEventObject -> = { +export const dataLayerActions: ActionFunctionMap = { assignChangedPages, - assignGatsbyNodeGraphQL, + assignGraphQLRunners, + callApi, + markNodesDirty, } diff --git a/packages/gatsby/src/state-machines/data-layer/index.ts b/packages/gatsby/src/state-machines/data-layer/index.ts index b814d2dece6b3..f1a02f30d0235 100644 --- a/packages/gatsby/src/state-machines/data-layer/index.ts +++ b/packages/gatsby/src/state-machines/data-layer/index.ts @@ -1,4 +1,4 @@ -import { MachineConfig, Machine } from "xstate" +import { Machine, StatesConfig, MachineOptions } from "xstate" import { dataLayerActions } from "./actions" import { IDataLayerContext } from "./types" import { dataLayerServices } from "./services" @@ -11,84 +11,182 @@ export type DataLayerResult = Pick< | "pagesToDelete" > -const dataLayerStates: MachineConfig = { - initial: `customizingSchema`, - states: { - customizingSchema: { - invoke: { - src: `customizeSchema`, - id: `customizing-schema`, - onDone: { - target: `sourcingNodes`, - }, +const loadDataStates: StatesConfig = { + customizingSchema: { + invoke: { + src: `customizeSchema`, + id: `customizing-schema`, + onDone: { + target: `sourcingNodes`, }, }, - sourcingNodes: { - invoke: { - src: `sourceNodes`, - id: `sourcing-nodes`, - onDone: { - target: `buildingSchema`, - actions: `assignChangedPages`, - }, + }, + sourcingNodes: { + invoke: { + src: `sourceNodes`, + id: `sourcing-nodes`, + onDone: { + target: `buildingSchema`, + actions: `assignChangedPages`, + }, + }, + }, +} + +const initialCreatePagesStates: StatesConfig = { + buildingSchema: { + invoke: { + id: `building-schema`, + src: `buildSchema`, + onDone: { + target: `creatingPages`, + actions: `assignGraphQLRunners`, + }, + }, + }, + creatingPages: { + on: { ADD_NODE_MUTATION: { actions: [`markNodesDirty`, `callApi`] } }, + invoke: { + id: `creating-pages`, + src: `createPages`, + onDone: { + target: `creatingPagesStatefully`, + actions: `assignChangedPages`, + }, + }, + }, + creatingPagesStatefully: { + invoke: { + src: `createPagesStatefully`, + id: `creating-pages-statefully`, + onDone: { + target: `rebuildingSchemaWithSitePage`, + }, + }, + }, + rebuildingSchemaWithSitePage: { + invoke: { + src: `rebuildSchemaWithSitePage`, + onDone: { + target: `writingOutRedirects`, }, }, - buildingSchema: { - invoke: { - id: `building-schema`, - src: `buildSchema`, - onDone: { - target: `creatingPages`, - actions: `assignGatsbyNodeGraphQL`, - }, + }, + writingOutRedirects: { + invoke: { + src: `writeOutRedirectsAndWatch`, + onDone: { + target: `done`, }, }, - creatingPages: { - invoke: { - id: `creating-pages`, - src: `createPages`, - onDone: [ - { - target: `creatingPagesStatefully`, - actions: `assignChangedPages`, - cond: (context): boolean => !!context.firstRun, - }, - { - target: `done`, - actions: `assignChangedPages`, - }, - ], + }, +} + +const recreatePagesStates: StatesConfig = { + buildingSchema: { + invoke: { + id: `building-schema`, + src: `buildSchema`, + onDone: { + target: `creatingPages`, + actions: `assignGraphQLRunners`, }, }, - creatingPagesStatefully: { - invoke: { - src: `createPagesStatefully`, - id: `creating-pages-statefully`, - onDone: { - target: `done`, - }, + }, + creatingPages: { + on: { ADD_NODE_MUTATION: { actions: [`markNodesDirty`, `callApi`] } }, + invoke: { + id: `creating-pages`, + src: `createPages`, + onDone: { + target: `rebuildingSchemaWithSitePage`, + actions: `assignChangedPages`, }, }, - done: { - type: `final`, - data: ({ + }, + rebuildingSchemaWithSitePage: { + invoke: { + src: `rebuildSchemaWithSitePage`, + onDone: { + target: `done`, + }, + }, + }, +} + +const doneState: StatesConfig = { + done: { + type: `final`, + data: ({ + gatsbyNodeGraphQLFunction, + graphqlRunner, + pagesToBuild, + pagesToDelete, + }): DataLayerResult => { + return { gatsbyNodeGraphQLFunction, graphqlRunner, pagesToBuild, pagesToDelete, - }): DataLayerResult => { - return { - gatsbyNodeGraphQLFunction, - graphqlRunner, - pagesToBuild, - pagesToDelete, - } - }, + } }, }, } -export const dataLayerMachine = Machine(dataLayerStates, { +const options: Partial> = { actions: dataLayerActions, services: dataLayerServices, -}) +} + +/** + * Machine used during first run + */ + +export const initializeDataMachine = Machine( + { + id: `initializeDataMachine`, + context: {}, + initial: `customizingSchema`, + states: { + ...loadDataStates, + ...initialCreatePagesStates, + ...doneState, + }, + }, + options +) + +/** + * Machine used when we need to source nodes again + */ + +export const reloadDataMachine = Machine( + { + id: `reloadDataMachine`, + context: {}, + initial: `customizingSchema`, + states: { + ...loadDataStates, + ...recreatePagesStates, + ...doneState, + }, + }, + options +) + +/** + * Machine used when we need to re-create pages after a + * node mutation outside of sourceNodes + */ +export const recreatePagesMachine = Machine( + { + id: `recreatePagesMachine`, + context: {}, + initial: `buildingSchema`, + states: { + ...recreatePagesStates, + ...doneState, + }, + }, + options +) diff --git a/packages/gatsby/src/state-machines/data-layer/services.ts b/packages/gatsby/src/state-machines/data-layer/services.ts index fd04e3c19df98..b8271a67b1107 100644 --- a/packages/gatsby/src/state-machines/data-layer/services.ts +++ b/packages/gatsby/src/state-machines/data-layer/services.ts @@ -5,6 +5,8 @@ import { createPagesStatefully, buildSchema, sourceNodes, + rebuildSchemaWithSitePage, + writeOutRedirects as writeOutRedirectsAndWatch, } from "../../services" import { IDataLayerContext } from "./types" @@ -17,4 +19,6 @@ export const dataLayerServices: Record< createPages, buildSchema, createPagesStatefully, + rebuildSchemaWithSitePage, + writeOutRedirectsAndWatch, } diff --git a/packages/gatsby/src/state-machines/data-layer/types.ts b/packages/gatsby/src/state-machines/data-layer/types.ts index 155c83021a059..3df3f4628e775 100644 --- a/packages/gatsby/src/state-machines/data-layer/types.ts +++ b/packages/gatsby/src/state-machines/data-layer/types.ts @@ -15,7 +15,8 @@ export interface IMutationAction { payload: unknown[] } export interface IDataLayerContext { - firstRun?: boolean + deferNodeMutation?: boolean + nodesMutatedDuringQueryRun?: boolean program?: IProgram store?: Store parentSpan?: Span diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts new file mode 100644 index 0000000000000..f2619fc6e76c0 --- /dev/null +++ b/packages/gatsby/src/state-machines/develop/actions.ts @@ -0,0 +1,140 @@ +import { + assign, + AnyEventObject, + ActionFunction, + spawn, + ActionFunctionMap, + DoneEventObject, +} from "xstate" +import { Store } from "redux" +import { IBuildContext, IMutationAction } from "../../services" +import { actions, boundActionCreators } from "../../redux/actions" +import { listenForMutations } from "../../services/listen-for-mutations" +import { DataLayerResult } from "../data-layer" +import { assertStore } from "../../utils/assert-store" +import { saveState } from "../../db" +import reporter from "gatsby-cli/lib/reporter" +import { ProgramStatus } from "../../redux/types" + +/** + * These are the deferred redux actions sent from api-runner-node + * They may include a `resolve` prop (if they are createNode actions). + * If so, we resolve the promise when we're done + */ +export const callRealApi = (event: IMutationAction, store?: Store): void => { + assertStore(store) + const { type, payload, resolve } = event + if (type in actions) { + // If this is a createNode action then this will be a thunk. + // No worries, we just dispatch it like any other + const action = actions[type](...payload) + const result = store.dispatch(action) + // Somebody may be waiting for this + if (resolve) { + resolve(result) + } + } else { + reporter.log(`Could not dispatch unknown action "${type}`) + } +} + +/** + * Handler for when we're inside handlers that should be able to mutate nodes + * Instead of queueing, we call it right away + */ +export const callApi: ActionFunction = ( + { store }, + event +) => callRealApi(event.payload, store) + +/** + * Event handler used in all states where we're not ready to process node + * mutations. Instead we add it to a batch to process when we're next idle + */ +export const addNodeMutation = assign({ + nodeMutationBatch: ({ nodeMutationBatch = [] }, { payload }) => { + // It's not pretty, but it's much quicker than concat + nodeMutationBatch.push(payload) + return nodeMutationBatch + }, +}) + +export const assignStoreAndWorkerPool = assign( + (_context, event) => { + const { store, workerPool } = event.data + return { + store, + workerPool, + } + } +) + +const setQueryRunningFinished = async (): Promise => { + boundActionCreators.setProgramStatus( + ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED + ) +} + +export const markQueryFilesDirty = assign({ + queryFilesDirty: true, +}) + +export const assignServiceResult = assign( + (_context, { data }): DataLayerResult => data +) + +/** + * This spawns the service that listens to the `emitter` for various mutation events + */ +export const spawnMutationListener = assign({ + mutationListener: () => spawn(listenForMutations, `listen-for-mutations`), +}) + +export const assignServers = assign( + (_context, { data }) => { + return { + ...data, + } + } +) + +export const assignWebhookBody = assign({ + webhookBody: (_context, { payload }) => payload?.webhookBody, +}) + +export const clearWebhookBody = assign({ + webhookBody: undefined, +}) + +export const finishParentSpan = ({ parentSpan }: IBuildContext): void => + parentSpan?.finish() + +export const saveDbState = (): Promise => saveState() + +/** + * Event handler used in all states where we're not ready to process a file change + * Instead we add it to a batch to process when we're next idle + */ +// export const markFilesDirty: BuildMachineAction = assign({ +// filesDirty: true, +// }) + +export const markNodesDirty = assign({ + nodesMutatedDuringQueryRun: true, +}) + +export const buildActions: ActionFunctionMap = { + callApi, + markNodesDirty, + addNodeMutation, + spawnMutationListener, + assignStoreAndWorkerPool, + assignServiceResult, + assignServers, + markQueryFilesDirty, + assignWebhookBody, + clearWebhookBody, + finishParentSpan, + saveDbState, + setQueryRunningFinished, +} diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts new file mode 100644 index 0000000000000..cd79e4d9831f4 --- /dev/null +++ b/packages/gatsby/src/state-machines/develop/index.ts @@ -0,0 +1,225 @@ +import { MachineConfig, AnyEventObject, forwardTo, Machine } from "xstate" +import { IDataLayerContext } from "../data-layer/types" +import { IQueryRunningContext } from "../query-running/types" +import { IWaitingContext } from "../waiting/types" +import { buildActions } from "./actions" +import { developServices } from "./services" +import { IBuildContext } from "../../services" + +/** + * This is the top-level state machine for the `gatsby develop` command + */ +const developConfig: MachineConfig = { + id: `build`, + initial: `initializing`, + // These are mutation events, sent to this machine by the mutation listener + // in `services/listen-for-mutations.ts` + on: { + // These are deferred node mutations, mainly `createNode` + ADD_NODE_MUTATION: { + actions: `addNodeMutation`, + }, + // Sent by query watcher, these are chokidar file events. They mean we + // need to extract queries + QUERY_FILE_CHANGED: { + actions: `markQueryFilesDirty`, + }, + // These are calls to the refresh endpoint. Also used by Gatsby Preview. + // Saves the webhook body from the event into context, then reloads data + WEBHOOK_RECEIVED: { + target: `reloadingData`, + actions: `assignWebhookBody`, + }, + }, + states: { + // Here we handle the initial bootstrap + initializing: { + on: { + // Ignore mutation events because we'll be running everything anyway + ADD_NODE_MUTATION: undefined, + QUERY_FILE_CHANGED: undefined, + WEBHOOK_RECEIVED: undefined, + }, + invoke: { + src: `initialize`, + onDone: { + target: `initializingData`, + actions: [`assignStoreAndWorkerPool`, `spawnMutationListener`], + }, + }, + }, + // Sourcing nodes, customising and inferring schema, then running createPages + initializingData: { + on: { + // We need to run mutations immediately when in this state + ADD_NODE_MUTATION: { + actions: [`markNodesDirty`, `callApi`], + }, + // Ignore, because we're about to extract them anyway + QUERY_FILE_CHANGED: undefined, + }, + invoke: { + src: `initializeData`, + data: ({ + parentSpan, + store, + webhookBody, + }: IBuildContext): IDataLayerContext => { + return { + parentSpan, + store, + webhookBody, + deferNodeMutation: true, + } + }, + onDone: { + actions: [ + `assignServiceResult`, + `clearWebhookBody`, + `finishParentSpan`, + ], + target: `runningQueries`, + }, + }, + }, + // Running page and static queries and generating the SSRed HTML and page data + runningQueries: { + on: { + QUERY_FILE_CHANGED: { + actions: forwardTo(`run-queries`), + }, + }, + invoke: { + id: `run-queries`, + src: `runQueries`, + // This is all the data that we're sending to the child machine + data: ({ + program, + store, + parentSpan, + gatsbyNodeGraphQLFunction, + graphqlRunner, + websocketManager, + }: IBuildContext): IQueryRunningContext => { + return { + program, + store, + parentSpan, + gatsbyNodeGraphQLFunction, + graphqlRunner, + websocketManager, + } + }, + onDone: [ + { + // If we have no compiler (i.e. it's first run), then spin up the + // webpack and socket.io servers + target: `startingDevServers`, + actions: `setQueryRunningFinished`, + cond: ({ compiler }: IBuildContext): boolean => !compiler, + }, + { + // ...otherwise just wait. + target: `waiting`, + }, + ], + }, + }, + // Spin up webpack and socket.io + startingDevServers: { + invoke: { + src: `startWebpackServer`, + onDone: { + target: `waiting`, + actions: `assignServers`, + }, + }, + }, + // Idle, waiting for events that make us rebuild + waiting: { + // We may want to save this is more places, but this should do for now + entry: `saveDbState`, + on: { + // Forward these events to the child machine, so it can handle batching + ADD_NODE_MUTATION: { + actions: forwardTo(`waiting`), + }, + QUERY_FILE_CHANGED: { + actions: forwardTo(`waiting`), + }, + // This event is sent from the child + EXTRACT_QUERIES_NOW: { + target: `runningQueries`, + }, + }, + invoke: { + id: `waiting`, + src: `waitForMutations`, + // Send existing queued mutations to the child machine, which will execute them + data: ({ + store, + nodeMutationBatch = [], + }: IBuildContext): IWaitingContext => { + return { store, nodeMutationBatch, runningBatch: [] } + }, + // "done" means we need to rebuild + onDone: { + actions: `assignServiceResult`, + target: `recreatingPages`, + }, + }, + }, + // Almost the same as initializing data, but skips various first-run stuff + reloadingData: { + on: { + // We need to run mutations immediately when in this state + ADD_NODE_MUTATION: { + actions: [`markNodesDirty`, `callApi`], + }, + // Ignore, because we're about to extract them anyway + QUERY_FILE_CHANGED: undefined, + }, + invoke: { + src: `reloadData`, + data: ({ + parentSpan, + store, + webhookBody, + }: IBuildContext): IDataLayerContext => { + return { + parentSpan, + store, + webhookBody, + deferNodeMutation: true, + } + }, + onDone: { + actions: [ + `assignServiceResult`, + `clearWebhookBody`, + `finishParentSpan`, + ], + target: `runningQueries`, + }, + }, + }, + // Rebuild pages if a node has been mutated outside of sourceNodes + recreatingPages: { + invoke: { + src: `recreatePages`, + data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { + return { parentSpan, store, deferNodeMutation: true } + }, + onDone: { + actions: `assignServiceResult`, + target: `runningQueries`, + }, + }, + }, + }, +} + +export const developMachine = Machine(developConfig, { + services: developServices, + actions: buildActions, +}) diff --git a/packages/gatsby/src/state-machines/develop/services.ts b/packages/gatsby/src/state-machines/develop/services.ts new file mode 100644 index 0000000000000..6f94baaebd00c --- /dev/null +++ b/packages/gatsby/src/state-machines/develop/services.ts @@ -0,0 +1,19 @@ +import { IBuildContext, startWebpackServer, initialize } from "../../services" +import { + initializeDataMachine, + reloadDataMachine, + recreatePagesMachine, +} from "../data-layer" +import { queryRunningMachine } from "../query-running" +import { waitingMachine } from "../waiting" +import { ServiceConfig } from "xstate" + +export const developServices: Record> = { + initializeData: initializeDataMachine, + reloadData: reloadDataMachine, + recreatePages: recreatePagesMachine, + initialize: initialize, + runQueries: queryRunningMachine, + waitForMutations: waitingMachine, + startWebpackServer: startWebpackServer, +} diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index c329e95ea8356..fe9013a4ee104 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -1,7 +1,5 @@ import { IQueryRunningContext } from "./types" import { DoneInvokeEvent, assign, ActionFunctionMap } from "xstate" -import { GraphQLRunner } from "../../query/graphql-runner" -import { assertStore } from "../../utils/assert-store" import { enqueueFlush } from "../../utils/page-data" export const flushPageData = (): void => { @@ -14,29 +12,14 @@ export const assignDirtyQueries = assign< >((_context, { data }) => { const { queryIds } = data return { - filesDirty: false, queryIds, } }) -export const resetGraphQLRunner = assign< - IQueryRunningContext, - DoneInvokeEvent ->({ - graphqlRunner: ({ store, program }) => { - assertStore(store) - return new GraphQLRunner(store, { - collectStats: true, - graphqlTracing: program?.graphqlTracing, - }) - }, -}) - export const queryActions: ActionFunctionMap< IQueryRunningContext, DoneInvokeEvent > = { - resetGraphQLRunner, assignDirtyQueries, flushPageData, } diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index 39161dac73304..7cc23417aae59 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -3,20 +3,23 @@ import { IQueryRunningContext } from "./types" import { queryRunningServices } from "./services" import { queryActions } from "./actions" +/** + * This is a child state machine, spawned to perform the query running + */ + export const queryStates: MachineConfig = { initial: `extractingQueries`, + id: `queryRunningMachine`, + context: {}, states: { extractingQueries: { id: `extracting-queries`, invoke: { id: `extracting-queries`, src: `extractQueries`, - onDone: [ - { - actions: `resetGraphQLRunner`, - target: `writingRequires`, - }, - ], + onDone: { + target: `writingRequires`, + }, }, }, writingRequires: { @@ -57,7 +60,7 @@ export const queryStates: MachineConfig = { }, }, }, - + // This waits for the jobs API to finish waitingForJobs: { invoke: { src: `waitUntilAllJobsComplete`, diff --git a/packages/gatsby/src/state-machines/waiting/actions.ts b/packages/gatsby/src/state-machines/waiting/actions.ts new file mode 100644 index 0000000000000..c5e4039705788 --- /dev/null +++ b/packages/gatsby/src/state-machines/waiting/actions.ts @@ -0,0 +1,32 @@ +import { + AssignAction, + assign, + ActionFunctionMap, + sendParent, + AnyEventObject, +} from "xstate" +import { IWaitingContext } from "./types" +import { AnyAction } from "redux" + +/** + * Event handler used when we're not ready to process node mutations. + * Instead we add it to a batch to process when we're next idle + */ +export const addNodeMutation: AssignAction = assign( + { + nodeMutationBatch: ({ nodeMutationBatch = [] }, { payload }) => { + // It's not pretty, but it's much quicker than concat + nodeMutationBatch.push(payload) + return nodeMutationBatch + }, + } +) + +export const extractQueries = sendParent( + `EXTRACT_QUERIES_NOW` +) + +export const waitingActions: ActionFunctionMap = { + addNodeMutation, + extractQueries, +} diff --git a/packages/gatsby/src/state-machines/waiting/index.ts b/packages/gatsby/src/state-machines/waiting/index.ts new file mode 100644 index 0000000000000..09d120a1df398 --- /dev/null +++ b/packages/gatsby/src/state-machines/waiting/index.ts @@ -0,0 +1,110 @@ +import { MachineConfig, assign, Machine } from "xstate" +import { IWaitingContext } from "./types" +import { waitingActions } from "./actions" +import { waitingServices } from "./services" + +const NODE_MUTATION_BATCH_SIZE = 100 +const NODE_MUTATION_BATCH_TIMEOUT = 1000 + +export type WaitingResult = Pick + +/** + * This idle state also handles batching of node mutations and running of + * mutations when we first start it + */ +export const waitingStates: MachineConfig = { + id: `waitingMachine`, + initial: `idle`, + context: { + nodeMutationBatch: [], + runningBatch: [], + }, + states: { + idle: { + always: { + // If we already have queued node mutations, move + // immediately to batching + cond: (ctx): boolean => !!ctx.nodeMutationBatch.length, + target: `batchingNodeMutations`, + }, + on: { + ADD_NODE_MUTATION: { + actions: `addNodeMutation`, + target: `batchingNodeMutations`, + }, + // We only listen for this when idling because if we receive it at any + // other point we're already going to create pages etc + QUERY_FILE_CHANGED: { + actions: `extractQueries`, + }, + }, + }, + + batchingNodeMutations: { + // Check if the batch is already full on entry + always: { + cond: (ctx): boolean => + ctx.nodeMutationBatch.length >= NODE_MUTATION_BATCH_SIZE, + target: `committingBatch`, + }, + on: { + // More mutations added to batch + ADD_NODE_MUTATION: [ + // You know the score: only run the first matching transition + { + // If this fills the batch then commit it + actions: `addNodeMutation`, + cond: (ctx): boolean => + ctx.nodeMutationBatch.length >= NODE_MUTATION_BATCH_SIZE, + target: `committingBatch`, + }, + { + // ...otherwise just add it to the batch + actions: `addNodeMutation`, + }, + ], + }, + after: { + // Time's up + [NODE_MUTATION_BATCH_TIMEOUT]: `committingBatch`, + }, + }, + committingBatch: { + entry: assign(({ nodeMutationBatch }) => { + return { + nodeMutationBatch: [], + runningBatch: nodeMutationBatch, + } + }), + on: { + // While we're running the batch we need to batch any incoming mutations too + ADD_NODE_MUTATION: { + actions: `addNodeMutation`, + }, + }, + invoke: { + src: `runMutationBatch`, + // When we're done, clear the running batch ready for next time + onDone: { + actions: assign({ + runningBatch: [], + }), + target: `rebuild`, + }, + }, + }, + rebuild: { + type: `final`, + // This is returned to the parent. The batch includes + // any mutations that arrived while we were running the other batch + data: ({ nodeMutationBatch }): WaitingResult => { + return { nodeMutationBatch } + }, + }, + }, +} + +export const waitingMachine = Machine(waitingStates, { + actions: waitingActions, + services: waitingServices, +}) diff --git a/packages/gatsby/src/state-machines/waiting/services.ts b/packages/gatsby/src/state-machines/waiting/services.ts new file mode 100644 index 0000000000000..f1362e07feaaa --- /dev/null +++ b/packages/gatsby/src/state-machines/waiting/services.ts @@ -0,0 +1,3 @@ +import { runMutationBatch } from "../../services" + +export const waitingServices = { runMutationBatch } diff --git a/packages/gatsby/src/state-machines/waiting/types.ts b/packages/gatsby/src/state-machines/waiting/types.ts new file mode 100644 index 0000000000000..0fbbc2a7240c0 --- /dev/null +++ b/packages/gatsby/src/state-machines/waiting/types.ts @@ -0,0 +1,14 @@ +import { Store, AnyAction } from "redux" +import { IGatsbyState } from "../../redux/types" + +export interface IMutationAction { + type: string + payload: unknown[] +} +export interface IWaitingContext { + nodeMutationBatch: IMutationAction[] + store?: Store + runningBatch: IMutationAction[] + filesDirty?: boolean + webhookBody?: Record +} diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index 6949bdfbb6784..4928a9fe38ba8 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -37,9 +37,11 @@ const { loadNodeContent } = require(`../db/nodes`) // metadata to actions they create. const boundPluginActionCreators = {} const doubleBind = (boundActionCreators, api, plugin, actionOptions) => { - const { traceId } = actionOptions - if (boundPluginActionCreators[plugin.name + api + traceId]) { - return boundPluginActionCreators[plugin.name + api + traceId] + const { traceId, deferNodeMutation } = actionOptions + const defer = deferNodeMutation ? `defer-node-mutation` : `` + const actionKey = plugin.name + api + traceId + defer + if (boundPluginActionCreators[actionKey]) { + return boundPluginActionCreators[actionKey] } else { const keys = Object.keys(boundActionCreators) const doubleBoundActionCreators = {} @@ -59,9 +61,7 @@ const doubleBind = (boundActionCreators, api, plugin, actionOptions) => { } } } - boundPluginActionCreators[ - plugin.name + api + traceId - ] = doubleBoundActionCreators + boundPluginActionCreators[actionKey] = doubleBoundActionCreators return doubleBoundActionCreators } } @@ -80,13 +80,55 @@ const initAPICallTracing = parentSpan => { } } +const deferredAction = type => (...args) => { + // Regular createNode returns a Promise, but when deferred we need + // to wrap it in another which we resolve when it's actually called + if (type === `createNode`) { + return new Promise(resolve => { + emitter.emit(`ENQUEUE_NODE_MUTATION`, { + type, + payload: args, + resolve, + }) + }) + } + return emitter.emit(`ENQUEUE_NODE_MUTATION`, { + type, + payload: args, + }) +} + +const NODE_MUTATION_ACTIONS = [ + `createNode`, + `deleteNode`, + `deleteNodes`, + `touchNode`, + `createParentChildLink`, + `createNodeField`, +] + +const deferActions = actions => { + const deferred = { ...actions } + NODE_MUTATION_ACTIONS.forEach(action => { + deferred[action] = deferredAction(action) + }) + return deferred +} + const getLocalReporter = (activity, reporter) => activity ? { ...reporter, panicOnBuild: activity.panicOnBuild.bind(activity) } : reporter +const pluginNodeCache = new Map() + const runAPI = async (plugin, api, args, activity) => { - const gatsbyNode = require(`${plugin.resolve}/gatsby-node`) + let gatsbyNode = pluginNodeCache.get(plugin.name) + if (!gatsbyNode) { + gatsbyNode = require(`${plugin.resolve}/gatsby-node`) + pluginNodeCache.set(plugin.name, gatsbyNode) + } + if (gatsbyNode[api]) { const parentSpan = args && args.parentSpan const spanOptions = parentSpan ? { childOf: parentSpan } : {} @@ -103,10 +145,15 @@ const runAPI = async (plugin, api, args, activity) => { ...publicActions, ...(restrictedActionsAvailableInAPI[api] || {}), } - const boundActionCreators = bindActionCreators( + let boundActionCreators = bindActionCreators( availableActions, store.dispatch ) + + if (args.deferNodeMutation) { + boundActionCreators = deferActions(boundActionCreators) + } + const doubleBoundActionCreators = doubleBind( boundActionCreators, api, diff --git a/packages/gatsby/src/utils/changed-pages.ts b/packages/gatsby/src/utils/changed-pages.ts new file mode 100644 index 0000000000000..8ca0836fa68bc --- /dev/null +++ b/packages/gatsby/src/utils/changed-pages.ts @@ -0,0 +1,54 @@ +import { boundActionCreators } from "../redux/actions" +const { deletePage, deleteComponentsDependencies } = boundActionCreators + +import { isEqualWith, IsEqualCustomizer } from "lodash" +import { IGatsbyPage } from "../redux/types" + +export function deleteUntouchedPages( + currentPages: Map, + timeBeforeApisRan: number +): string[] { + const deletedPages: string[] = [] + + // Delete pages that weren't updated when running createPages. + currentPages.forEach(page => { + if ( + !page.isCreatedByStatefulCreatePages && + page.updatedAt < timeBeforeApisRan && + page.path !== `/404.html` + ) { + deleteComponentsDependencies([page.path]) + deletePage(page) + deletedPages.push(page.path, `/page-data${page.path}`) + } + }) + return deletedPages +} + +export function findChangedPages( + oldPages: Map, + currentPages: Map +): { + changedPages: string[] + deletedPages: string[] +} { + const changedPages: string[] = [] + + const compareWithoutUpdated: IsEqualCustomizer = (_left, _right, key) => + key === `updatedAt` || undefined + + currentPages.forEach((newPage, path) => { + const oldPage = oldPages.get(path) + if (!oldPage || !isEqualWith(newPage, oldPage, compareWithoutUpdated)) { + changedPages.push(path) + } + }) + const deletedPages: string[] = [] + oldPages.forEach((_page, key) => { + if (!currentPages.has(key)) { + deletedPages.push(key) + } + }) + + return { changedPages, deletedPages } +} diff --git a/packages/gatsby/src/utils/source-nodes.ts b/packages/gatsby/src/utils/source-nodes.ts index 68329e2f7e092..fb9b982a27388 100644 --- a/packages/gatsby/src/utils/source-nodes.ts +++ b/packages/gatsby/src/utils/source-nodes.ts @@ -85,13 +85,16 @@ function deleteStaleNodes(state: IGatsbyState, nodes: Node[]): void { export default async ({ webhookBody, parentSpan, + deferNodeMutation = false, }: { - webhookBody?: unknown - parentSpan?: Span + webhookBody: unknown + parentSpan: Span + deferNodeMutation: boolean }): Promise => { await apiRunner(`sourceNodes`, { traceId: `initial-sourceNodes`, waitForCascadingActions: true, + deferNodeMutation, parentSpan, webhookBody: webhookBody || {}, }) diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts index 4708b8278219f..d1d2be626d2de 100644 --- a/packages/gatsby/src/utils/start-server.ts +++ b/packages/gatsby/src/utils/start-server.ts @@ -13,7 +13,7 @@ import graphiqlExplorer from "gatsby-graphiql-explorer" import { formatError } from "graphql" import webpackConfig from "../utils/webpack.config" -import { store } from "../redux" +import { store, emitter } from "../redux" import { buildHTML } from "../commands/build-html" import { withBasePath } from "../utils/path" import report from "gatsby-cli/lib/reporter" @@ -34,14 +34,6 @@ import { Express } from "express" import { Stage, IProgram } from "../commands/types" import JestWorker from "jest-worker" -import { - startSchemaHotReloader, - stopSchemaHotReloader, -} from "../bootstrap/schema-hot-reloader" - -import sourceNodes from "../utils/source-nodes" -import { createSchemaCustomization } from "../utils/create-schema-customization" -import { rebuild as rebuildSchema } from "../schema" type ActivityTracker = any // TODO: Replace this with proper type once reporter is typed interface IServer { @@ -53,7 +45,7 @@ interface IServer { webpackWatching: IWebpackWatchingPauseResume } -interface IWebpackWatchingPauseResume { +export interface IWebpackWatchingPauseResume extends webpack.Watching { suspend: () => void resume: () => void } @@ -63,7 +55,7 @@ interface IWebpackWatchingPauseResume { type PatchedWebpackDevMiddleware = WebpackDevMiddleware & express.RequestHandler & { context: { - watching: webpack.Watching & IWebpackWatchingPauseResume + watching: IWebpackWatchingPauseResume } } @@ -187,31 +179,15 @@ export async function startServer( ) /** - * This will be removed in state machine * Refresh external data sources. * This behavior is disabled by default, but the ENABLE_GATSBY_REFRESH_ENDPOINT env var enables it * If no GATSBY_REFRESH_TOKEN env var is available, then no Authorization header is required **/ const REFRESH_ENDPOINT = `/__refresh` const refresh = async (req: express.Request): Promise => { - stopSchemaHotReloader() - let activity = report.activityTimer(`createSchemaCustomization`, {}) - activity.start() - await createSchemaCustomization({ - refresh: true, - }) - activity.end() - activity = report.activityTimer(`Refreshing source data`, {}) - activity.start() - await sourceNodes({ + emitter.emit(`WEBHOOK_RECEIVED`, { webhookBody: req.body, }) - activity.end() - activity = report.activityTimer(`rebuild schema`) - activity.start() - await rebuildSchema({ parentSpan: activity }) - activity.end() - startSchemaHotReloader() } app.use(REFRESH_ENDPOINT, express.json()) app.post(REFRESH_ENDPOINT, (req, res) => { diff --git a/packages/gatsby/src/utils/state-machine-logging.ts b/packages/gatsby/src/utils/state-machine-logging.ts new file mode 100644 index 0000000000000..9e4778c08e375 --- /dev/null +++ b/packages/gatsby/src/utils/state-machine-logging.ts @@ -0,0 +1,53 @@ +import { + DefaultContext, + Interpreter, + Actor, + State, + AnyEventObject, +} from "xstate" +import reporter from "gatsby-cli/lib/reporter" + +const isInterpreter = ( + actor: Actor | Interpreter +): actor is Interpreter => `machine` in actor + +export function logTransitions( + service: Interpreter +): void { + const listeners = new WeakSet() + let last: State + + service.onTransition(state => { + if (!last) { + last = state + } else if (!state.changed || last.matches(state)) { + return + } + last = state + reporter.verbose(`Transition to ${JSON.stringify(state.value)}`) + // eslint-disable-next-line no-unused-expressions + service.children?.forEach(child => { + // We want to ensure we don't attach a listener to the same + // actor. We don't need to worry about detaching the listener + // because xstate handles that for us when the actor is stopped. + + if (isInterpreter(child) && !listeners.has(child)) { + let sublast = child.state + child.onTransition(substate => { + if (!sublast) { + sublast = substate + } else if (!substate.changed || sublast.matches(substate)) { + return + } + sublast = substate + reporter.verbose( + `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify( + substate.value + )}` + ) + }) + listeners.add(child) + } + }) + }) +}