Handle webpack in state machine (#25815)

Co-authored-by: gatsbybot <mathews.kyle+gatsbybot@gmail.com>

packages/gatsby/src/commands/develop-process.ts

@@ -69,9 +69,7 @@ const openDebuggerPort = (debugInfo: IDebugInfo): void => {
 }
 
 module.exports = async (program: IDevelopArgs): Promise<void> => {
-  if (program.verbose) {
-    reporter.setVerbose(true)
-  }
+  reporter.setVerbose(program.verbose)
 
   if (program.debugInfo) {
     openDebuggerPort(program.debugInfo)

packages/gatsby/src/query/query-watcher.js

@@ -237,13 +237,13 @@ const watch = async rootDir => {
       { ignoreInitial: true }
     )
     .on(`change`, path => {
-      emitter.emit(`QUERY_FILE_CHANGED`, path)
+      emitter.emit(`SOURCE_FILE_CHANGED`, path)
     })
     .on(`add`, path => {
-      emitter.emit(`QUERY_FILE_CHANGED`, path)
+      emitter.emit(`SOURCE_FILE_CHANGED`, path)
     })
     .on(`unlink`, path => {
-      emitter.emit(`QUERY_FILE_CHANGED`, path)
+      emitter.emit(`SOURCE_FILE_CHANGED`, path)
     })
 
   filesToWatch.forEach(filePath => watcher.add(filePath))

packages/gatsby/src/services/index.ts

@@ -19,6 +19,7 @@ import { runPageQueries } from "./run-page-queries"
 
 import { waitUntilAllJobsComplete } from "../utils/wait-until-jobs-complete"
 import { runMutationBatch } from "./run-mutation-batch"
+import { recompile } from "./recompile"
 
 export * from "./types"
 
@@ -40,6 +41,7 @@ export {
   startWebpackServer,
   rebuildSchemaWithSitePage,
   runMutationBatch,
+  recompile,
 }
 
 export const buildServices: Record<string, ServiceConfig<IBuildContext>> = {
@@ -59,4 +61,5 @@ export const buildServices: Record<string, ServiceConfig<IBuildContext>> = {
   writeOutRedirects,
   startWebpackServer,
   rebuildSchemaWithSitePage,
+  recompile,
 }

packages/gatsby/src/services/listen-for-mutations.ts

@@ -6,27 +6,21 @@ export const listenForMutations: InvokeCallback = (callback: Sender<any>) => {
     callback({ type: `ADD_NODE_MUTATION`, payload: event })
   }
 
-  const emitFileChange = (event: unknown): void => {
+  const emitSourceChange = (event: unknown): void => {
     callback({ type: `SOURCE_FILE_CHANGED`, payload: event })
   }
 
-  const emitQueryChange = (event: unknown): void => {
-    callback({ type: `QUERY_FILE_CHANGED`, payload: event })
-  }
-
   const emitWebhook = (event: unknown): void => {
     callback({ type: `WEBHOOK_RECEIVED`, payload: event })
   }
 
   emitter.on(`ENQUEUE_NODE_MUTATION`, emitMutation)
   emitter.on(`WEBHOOK_RECEIVED`, emitWebhook)
-  emitter.on(`SOURCE_FILE_CHANGED`, emitFileChange)
-  emitter.on(`QUERY_FILE_CHANGED`, emitQueryChange)
+  emitter.on(`SOURCE_FILE_CHANGED`, emitSourceChange)
 
   return function unsubscribeFromMutationListening(): void {
     emitter.off(`ENQUEUE_NODE_MUTATION`, emitMutation)
-    emitter.off(`SOURCE_FILE_CHANGED`, emitFileChange)
     emitter.off(`WEBHOOK_RECEIVED`, emitWebhook)
-    emitter.off(`QUERY_FILE_CHANGED`, emitQueryChange)
+    emitter.off(`SOURCE_FILE_CHANGED`, emitSourceChange)
   }
 }

packages/gatsby/src/services/listen-to-webpack.ts

@@ -0,0 +1,12 @@
+import { Compiler } from "webpack"
+import { InvokeCallback } from "xstate"
+import reporter from "gatsby-cli/lib/reporter"
+
+export const createWebpackWatcher = (compiler: Compiler): InvokeCallback => (
+  callback
+): void => {
+  compiler.hooks.invalid.tap(`file invalidation`, file => {
+    reporter.verbose(`Webpack file changed: ${file}`)
+    callback({ type: `SOURCE_FILE_CHANGED`, file })
+  })
+}

packages/gatsby/src/services/recompile.ts

@@ -0,0 +1,27 @@
+/* eslint-disable no-unused-expressions */
+import { IBuildContext } from "./types"
+import { Stats } from "webpack"
+import reporter from "gatsby-cli/lib/reporter"
+import { emitter } from "../redux"
+
+export async function recompile({
+  webpackWatching,
+}: IBuildContext): Promise<Stats> {
+  if (!webpackWatching) {
+    reporter.panic(`Missing compiler`)
+  }
+  // Promisify the event-based API. We do this using emitter
+  // because you can't "untap" a webpack watcher, and we just want
+  // one compilation.
+
+  return new Promise(resolve => {
+    function finish(stats: Stats): void {
+      emitter.off(`COMPILATION_DONE`, finish)
+      resolve(stats)
+    }
+    emitter.on(`COMPILATION_DONE`, finish)
+    webpackWatching.resume()
+    // Suspending is just a flag, so it's safe to re-suspend right away
+    webpackWatching.suspend()
+  })
+}

packages/gatsby/src/services/start-webpack-server.ts

@@ -22,6 +22,7 @@ import {
 } from "../utils/webpack-status"
 import { enqueueFlush } from "../utils/page-data"
 import mapTemplatesToStaticQueryHashes from "../utils/map-templates-to-static-query-hashes"
+import { emitter } from "../redux"
 
 export async function startWebpackServer({
   program,
@@ -42,6 +43,7 @@ export async function startWebpackServer({
     websocketManager,
     webpackWatching,
   } = await startServer(program, app, workerPool)
+  webpackWatching.suspend()
 
   compiler.hooks.invalid.tap(`log compiling`, function () {
     if (!webpackActivity) {
@@ -160,6 +162,7 @@ export async function startWebpackServer({
 
       markWebpackStatusAsDone()
       done()
+      emitter.emit(`COMPILATION_DONE`, stats)
       resolve({ compiler, websocketManager, webpackWatching })
     })
   })

packages/gatsby/src/services/types.ts

@@ -37,5 +37,7 @@ export interface IBuildContext {
   compiler?: Compiler
   websocketManager?: WebsocketManager
   webpackWatching?: IWebpackWatchingPauseResume
+  webpackListener?: Actor<unknown, AnyEventObject>
   queryFilesDirty?: boolean
+  sourceFilesDirty?: boolean
 }

packages/gatsby/src/state-machines/develop/actions.ts

@@ -15,6 +15,7 @@ import { assertStore } from "../../utils/assert-store"
 import { saveState } from "../../db"
 import reporter from "gatsby-cli/lib/reporter"
 import { ProgramStatus } from "../../redux/types"
+import { createWebpackWatcher } from "../../services/listen-to-webpack"
 
 /**
  * These are the deferred redux actions sent from api-runner-node
@@ -79,6 +80,14 @@ export const markQueryFilesDirty = assign<IBuildContext>({
   queryFilesDirty: true,
 })
 
+export const markSourceFilesDirty = assign<IBuildContext>({
+  sourceFilesDirty: true,
+})
+
+export const markSourceFilesClean = assign<IBuildContext>({
+  sourceFilesDirty: false,
+})
+
 export const assignServiceResult = assign<IBuildContext, DoneEventObject>(
   (_context, { data }): DataLayerResult => data
 )
@@ -98,6 +107,15 @@ export const assignServers = assign<IBuildContext, AnyEventObject>(
   }
 )
 
+export const spawnWebpackListener = assign<IBuildContext, AnyEventObject>({
+  webpackListener: ({ compiler }) => {
+    if (!compiler) {
+      return undefined
+    }
+    return spawn(createWebpackWatcher(compiler))
+  },
+})
+
 export const assignWebhookBody = assign<IBuildContext, AnyEventObject>({
   webhookBody: (_context, { payload }) => payload?.webhookBody,
 })
@@ -135,6 +153,9 @@ export const buildActions: ActionFunctionMap<IBuildContext, AnyEventObject> = {
   assignWebhookBody,
   clearWebhookBody,
   finishParentSpan,
+  spawnWebpackListener,
+  markSourceFilesDirty,
+  markSourceFilesClean,
   saveDbState,
   setQueryRunningFinished,
 }

packages/gatsby/src/state-machines/develop/index.ts

@@ -19,10 +19,9 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
     ADD_NODE_MUTATION: {
       actions: `addNodeMutation`,
     },
-    // Sent by query watcher, these are chokidar file events. They mean we
-    // need to extract queries
-    QUERY_FILE_CHANGED: {
-      actions: `markQueryFilesDirty`,
+    // Sent when webpack or chokidar sees a changed file
+    SOURCE_FILE_CHANGED: {
+      actions: `markSourceFilesDirty`,
     },
     // These are calls to the refresh endpoint. Also used by Gatsby Preview.
     // Saves the webhook body from the event into context, then reloads data
@@ -37,7 +36,7 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
       on: {
         // Ignore mutation events because we'll be running everything anyway
         ADD_NODE_MUTATION: undefined,
-        QUERY_FILE_CHANGED: undefined,
+        SOURCE_FILE_CHANGED: undefined,
         WEBHOOK_RECEIVED: undefined,
       },
       invoke: {
@@ -55,8 +54,6 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
         ADD_NODE_MUTATION: {
           actions: [`markNodesDirty`, `callApi`],
         },
-        // Ignore, because we're about to extract them anyway
-        QUERY_FILE_CHANGED: undefined,
       },
       invoke: {
         src: `initializeData`,
@@ -85,8 +82,8 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
     // Running page and static queries and generating the SSRed HTML and page data
     runningQueries: {
       on: {
-        QUERY_FILE_CHANGED: {
-          actions: forwardTo(`run-queries`),
+        SOURCE_FILE_CHANGED: {
+          actions: [forwardTo(`run-queries`), `markSourceFilesDirty`],
         },
       },
       invoke: {
@@ -118,34 +115,53 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
             actions: `setQueryRunningFinished`,
             cond: ({ compiler }: IBuildContext): boolean => !compiler,
           },
+          {
+            // If source files have changed, then recompile the JS bundle
+            target: `recompiling`,
+            cond: ({ sourceFilesDirty }: IBuildContext): boolean =>
+              !!sourceFilesDirty,
+          },
           {
             // ...otherwise just wait.
             target: `waiting`,
           },
         ],
       },
     },
+    // Recompile the JS bundle
+    recompiling: {
+      invoke: {
+        src: `recompile`,
+        onDone: {
+          actions: `markSourceFilesClean`,
+          target: `waiting`,
+        },
+      },
+    },
     // Spin up webpack and socket.io
     startingDevServers: {
       invoke: {
         src: `startWebpackServer`,
         onDone: {
           target: `waiting`,
-          actions: `assignServers`,
+          actions: [
+            `assignServers`,
+            `spawnWebpackListener`,
+            `markSourceFilesClean`,
+          ],
         },
       },
     },
     // Idle, waiting for events that make us rebuild
     waiting: {
-      // We may want to save this is more places, but this should do for now
       entry: `saveDbState`,
       on: {
         // Forward these events to the child machine, so it can handle batching
         ADD_NODE_MUTATION: {
           actions: forwardTo(`waiting`),
         },
-        QUERY_FILE_CHANGED: {
-          actions: forwardTo(`waiting`),
+        SOURCE_FILE_CHANGED: {
+          actions: [forwardTo(`waiting`), `markSourceFilesDirty`],
         },
         // This event is sent from the child
         EXTRACT_QUERIES_NOW: {
@@ -177,7 +193,7 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
           actions: [`markNodesDirty`, `callApi`],
         },
         // Ignore, because we're about to extract them anyway
-        QUERY_FILE_CHANGED: undefined,
+        SOURCE_FILE_CHANGED: undefined,
       },
       invoke: {
         src: `reloadData`,

packages/gatsby/src/state-machines/develop/services.ts

@@ -1,4 +1,9 @@
-import { IBuildContext, startWebpackServer, initialize } from "../../services"
+import {
+  IBuildContext,
+  startWebpackServer,
+  initialize,
+  recompile,
+} from "../../services"
 import {
   initializeDataMachine,
   reloadDataMachine,
@@ -15,5 +20,6 @@ export const developServices: Record<string, ServiceConfig<IBuildContext>> = {
   initialize: initialize,
   runQueries: queryRunningMachine,
   waitForMutations: waitingMachine,
-  startWebpackServer: startWebpackServer,
+  startWebpackServer,
+  recompile,
 }

packages/gatsby/src/state-machines/query-running/index.ts

@@ -10,6 +10,11 @@ import { queryActions } from "./actions"
 export const queryStates: MachineConfig<IQueryRunningContext, any, any> = {
   initial: `extractingQueries`,
   id: `queryRunningMachine`,
+  on: {
+    SOURCE_FILE_CHANGED: {
+      target: `extractingQueries`,
+    },
+  },
   context: {},
   states: {
     extractingQueries: {

packages/gatsby/src/state-machines/waiting/index.ts

@@ -34,12 +34,33 @@ export const waitingStates: MachineConfig<IWaitingContext, any, any> = {
         },
         // We only listen for this when idling because if we receive it at any
         // other point we're already going to create pages etc
-        QUERY_FILE_CHANGED: {
+        SOURCE_FILE_CHANGED: {
+          target: `aggregatingFileChanges`,
+        },
+      },
+    },
+    aggregatingFileChanges: {
+      // Sigh. This is because webpack doesn't expose the Watchpack
+      // aggregated file invalidation events. If we compile immediately,
+      // we won't pick up the changed files
+      after: {
+        // The aggregation timeout
+        200: {
           actions: `extractQueries`,
+          target: `idle`,
+        },
+      },
+      on: {
+        ADD_NODE_MUTATION: {
+          actions: `addNodeMutation`,
+        },
+        SOURCE_FILE_CHANGED: {
+          target: undefined,
+          // External self-transition reset the timer
+          internal: false,
         },
       },
     },
-
     batchingNodeMutations: {
       // Check if the batch is already full on entry
       always: {