move query/pages-writer to bootstrap/requires-writer (#13729)

And delete data.json

packages/gatsby/src/bootstrap/__tests__/requires-writer.js

@@ -0,0 +1,47 @@
+const { joinPath } = require(`../../utils/path`)
+const requiresWriter = require(`../requires-writer`)
+
+const now = Date.now()
+
+const newMockState = () => {
+  const pages = new Map()
+  pages.set(`path1`, {
+    component: `component1`,
+    componentChunkName: `chunkName1`,
+    matchPath: `matchPath1`,
+    path: `/path1`,
+  })
+  pages.set(`path2`, {
+    component: `component2`,
+    componentChunkName: `chunkName2`,
+    path: `/path2`,
+  })
+  const program = { directory: `/dir` }
+  return { pages, program }
+}
+
+jest.mock(`fs-extra`, () => {
+  return {
+    writeFile: () => Promise.resolve(),
+    move: () => {},
+  }
+})
+
+const mockFsExtra = require(`fs-extra`)
+
+describe(`requires-writer`, () => {
+  beforeEach(() => {
+    global.Date.now = () => now
+    requiresWriter.resetLastHash()
+  })
+  describe(`writeAll`, () => {
+    it(`writes requires files`, async () => {
+      const spy = jest.spyOn(mockFsExtra, `writeFile`)
+      await requiresWriter.writeAll(newMockState())
+      expect(spy).toBeCalledWith(
+        joinPath(`/dir`, `.cache`, `match-paths.json.${now}`),
+        JSON.stringify([{ path: `/path1`, matchPath: `matchPath1` }], null, 4)
+      )
+    })
+  })
+})

packages/gatsby/src/bootstrap/index.js

@@ -32,7 +32,7 @@ process.on(`unhandledRejection`, (reason, p) => {
 
 const { extractQueries } = require(`../query/query-watcher`)
 const queryUtil = require(`../query`)
-const { writePages } = require(`../query/pages-writer`)
+const requiresWriter = require(`./requires-writer`)
 const { writeRedirects } = require(`./redirects-writer`)
 
 // Override console.log to add the source file + line number.
@@ -480,7 +480,7 @@ module.exports = async (args: BootstrapArgs) => {
   })
   activity.start()
   try {
-    await writePages()
+    await requiresWriter.writeAll(store.getState())
   } catch (err) {
     report.panic(`Failed to write out page data`, err)
   }

packages/gatsby/src/bootstrap/requires-writer.js

@@ -0,0 +1,126 @@
+const _ = require(`lodash`)
+const fs = require(`fs-extra`)
+const crypto = require(`crypto`)
+const { store, emitter } = require(`../redux/`)
+import { joinPath } from "../utils/path"
+
+let lastHash = null
+
+const resetLastHash = () => {
+  lastHash = null
+}
+
+const pickComponentFields = page =>
+  _.pick(page, [`component`, `componentChunkName`])
+
+const getComponents = pages =>
+  _(pages)
+    .map(pickComponentFields)
+    .uniqBy(c => c.componentChunkName)
+    .value()
+
+const pickMatchPathFields = page => _.pick(page, [`path`, `matchPath`])
+
+const getMatchPaths = pages =>
+  pages.filter(page => page.matchPath).map(pickMatchPathFields)
+
+const createHash = (matchPaths, components) =>
+  crypto
+    .createHash(`md5`)
+    .update(JSON.stringify({ matchPaths, components }))
+    .digest(`hex`)
+
+// Write out pages information.
+const writeAll = async state => {
+  const { program } = state
+  const pages = [...state.pages.values()]
+  const matchPaths = getMatchPaths(pages)
+  const components = getComponents(pages)
+
+  const newHash = createHash(matchPaths, components)
+
+  if (newHash === lastHash) {
+    // Nothing changed. No need to rewrite files
+    return Promise.resolve()
+  }
+
+  lastHash = newHash
+
+  // Create file with sync requires of components/json files.
+  let syncRequires = `const { hot } = require("react-hot-loader/root")
+
+// prefer default export if available
+const preferDefault = m => m && m.default || m
+\n\n`
+  syncRequires += `exports.components = {\n${components
+    .map(
+      c =>
+        `  "${c.componentChunkName}": hot(preferDefault(require("${joinPath(
+          c.component
+        )}")))`
+    )
+    .join(`,\n`)}
+}\n\n`
+
+  // Create file with async requires of components/json files.
+  let asyncRequires = `// prefer default export if available
+const preferDefault = m => m && m.default || m
+\n`
+  asyncRequires += `exports.components = {\n${components
+    .map(
+      c =>
+        `  "${c.componentChunkName}": () => import("${joinPath(
+          c.component
+        )}" /* webpackChunkName: "${c.componentChunkName}" */)`
+    )
+    .join(`,\n`)}
+}\n\n`
+
+  const writeAndMove = (file, data) => {
+    const destination = joinPath(program.directory, `.cache`, file)
+    const tmp = `${destination}.${Date.now()}`
+    return fs
+      .writeFile(tmp, data)
+      .then(() => fs.move(tmp, destination, { overwrite: true }))
+  }
+
+  const result = await Promise.all([
+    writeAndMove(`sync-requires.js`, syncRequires),
+    writeAndMove(`async-requires.js`, asyncRequires),
+    writeAndMove(`match-paths.json`, JSON.stringify(matchPaths, null, 4)),
+  ])
+
+  return result
+}
+
+const debouncedWriteAll = _.debounce(() => writeAll(store.getState()), 500, {
+  leading: true,
+})
+
+/**
+ * Start listening to CREATE/DELETE_PAGE events so we can rewrite
+ * files as required
+ */
+const startListener = () => {
+  emitter.on(`CREATE_PAGE`, () => {
+    debouncedWriteAll()
+  })
+
+  emitter.on(`CREATE_PAGE_END`, () => {
+    debouncedWriteAll()
+  })
+
+  emitter.on(`DELETE_PAGE`, () => {
+    debouncedWriteAll()
+  })
+
+  emitter.on(`DELETE_PAGE_BY_PATH`, () => {
+    debouncedWriteAll()
+  })
+}
+
+module.exports = {
+  writeAll,
+  resetLastHash,
+  startListener,
+}

packages/gatsby/src/commands/develop.js

@@ -38,6 +38,7 @@ const onExit = require(`signal-exit`)
 const queryUtil = require(`../query`)
 const queryQueue = require(`../query/queue`)
 const queryWatcher = require(`../query/query-watcher`)
+const requiresWriter = require(`../bootstrap/requires-writer`)
 
 // const isInteractive = process.stdout.isTTY
 
@@ -93,6 +94,7 @@ async function startServer(program) {
 
   db.startAutosave()
   queryUtil.startListening(queryQueue.createDevelopQueue())
+  requiresWriter.startListener()
   queryWatcher.startWatchDeletePage()
 
   await createIndexHtml()