Explore page build optimisations

docs/docs/incremental-builds.md

@@ -0,0 +1,61 @@
+---
+title: Incremental Builds
+---
+
+Gatsby sources data from multiple sources (CMS, static files like Markdown, databases, APIs etc) and creates an aggregated data set in GraphQL. Currently, each `gatsby build` uses the GraphQL data set and queries to do a complete rebuild of the whole app, including static assets, such as HTML, JavaScript, JSON, and media files etc) from the graphQL queries, ready for deployment. For projects that have a small (10s to 100s) to medium (100s to 1000s) amount of content, these full builds don't present a problem.
+
+Sites with large amounts of content (10,000s upwards 😱) start to see increased build times and increased demand on CPU and memory.
+
+Also, one of the principals of modern Continuous Integration/Continuous Deployment is to release change (and therefore risk) in small batches. A full app rebuild may not be in line with that principle for some projects.
+
+One solution to these problems might be to use [Gatsby Cloud](https://www.gatsbyjs.com/cloud/)'s 'Build' features (currently in [Beta](https://www.gatsbyjs.com/builds-beta/)).
+
+For projects that require self-hosted environments, where Gatsby Cloud would not be an option, being able to **incrementally build** only the content that has changed (or new) would help reduce build times and demand on resources, whilst helping keep in line with CI/CD principles.
+
+For more info on the standard build process please see [overview of the gatsby build process](/docs/overview-of-the-gatsby-build-process/)
+
+## How to use
+
+To enable optional incremental builds, use the environment variable `GATSBY_INCREMENTAL_BUILD=true` in your `gatsby build` command, for example:
+
+`GATSBY_INCREMENTAL_BUILD=true gatsby build`
+
+This will run the Gatsby build process, but only build assets that have changed (or are new) since your last build.
+
+### Reporting what has been built
+
+After an incremental build has completed, you might need to get a list of the assets that have been built, for example, if you want to perform a sync action in your CI/CD pipeline.
+
+To list the paths in the build assets (`public`) folder, you can use one (or both) of the following arguments in your `build` command.
+
+- `--log-pages` outputs the updated paths to the console at the end of the build
+
+```
+success Building production JavaScript and CSS bundles - 82.198s
+success run queries - 82.762s - 4/4 0.05/s
+success Building static HTML for pages - 19.386s - 2/2 0.10/s
++ success Delete previous page data - 1.512s
+success Update cache for next build - 1.202s
+info Done building in 152.084 sec
++ info Incremental build pages:
++ Updated page: /about
++ Updated page: /accounts/example
++ info Incremental build deleted pages:
++ Deleted page: /test
+
+Done in 154.501 sec
+```
+
+- `--write-to-file` creates two files in the `.cache` folder, with lists of the changes paths in the build assets (`public`) folder.
+
+  - `newPages.txt` will contain a list of paths that have changed or are new
+  - `deletedPages.txt` will contain a list of paths that have been deleted
+
+If there are no changed or deleted paths, then the relevant files will not be created in the `.cache` folder.
+
+## Further considerations
+
+- To enable incremental builds you will need to set an environment variable, so you will need access to set variables in your build environment
+- You will need to persist the cached `.cache/redux.state` file between builds, allowing for comparison on an incremental build, if there is no `redux.state` file located in the `.cache` the folder then a full build will be triggered
+- The root JS bundle will still get generated on incremental builds, you will need to deploy these JS files as well as changed paths
+- Any code changes (templates, components, source handling, new plugins etc) will require a full `gatsby build`

docs/docs/scaling-issues.md

@@ -88,3 +88,7 @@ exports.createSchemaCustomization = ({ actions }) => {
   `)
 }
 ```
+
+### Incremental builds
+
+Being able to incrementally build only the content that has changed (or new) would help reduce build times and demand on resources. See [Incremental builds](/docs/incremental-builds/).

packages/gatsby/src/bootstrap/index.js

@@ -23,6 +23,9 @@ const removeStaleJobs = require(`./remove-stale-jobs`)
 // Add `util.promisify` polyfill for old node versions
 require(`util.promisify/shim`)()
 
+const incrementalBuild =
+  process.env.GATSBY_INCREMENTAL_BUILD === `true` || false
+
 // Show stack trace on unhandled promises.
 process.on(`unhandledRejection`, (reason, p) => {
   report.panic(reason)
@@ -190,7 +193,7 @@ module.exports = async (args: BootstrapArgs) => {
 
   // During builds, delete html and css files from the public directory as we don't want
   // deleted pages and styles from previous builds to stick around.
-  if (process.env.NODE_ENV === `production`) {
+  if (!incrementalBuild && process.env.NODE_ENV === `production`) {
     activity = report.activityTimer(
       `delete html and css files from previous builds`,
       {
@@ -252,7 +255,12 @@ module.exports = async (args: BootstrapArgs) => {
     try {
       // Attempt to empty dir if remove fails,
       // like when directory is mount point
-      await fs.remove(cacheDirectory).catch(() => fs.emptyDir(cacheDirectory))
+      if (!incrementalBuild) {
+        await fs.remove(cacheDirectory).catch(() => fs.emptyDir(cacheDirectory))
+      } else {
+        // Remove all files except the cache file
+        await del([`${cacheDirectory}/**/*.{json,js,css}`])
+      }
     } catch (e) {
       report.error(`Failed to remove .cache files.`, e)
     }

packages/gatsby/src/commands/build.js

@@ -1,6 +1,6 @@
 /* @flow */
-
 const path = require(`path`)
+const fs = require(`fs-extra`)
 const report = require(`gatsby-cli/lib/reporter`)
 const buildHTML = require(`./build-html`)
 const buildProductionBundle = require(`./build-javascript`)
@@ -11,11 +11,12 @@ const { initTracer, stopTracer } = require(`../utils/tracer`)
 const db = require(`../db`)
 const signalExit = require(`signal-exit`)
 const telemetry = require(`gatsby-telemetry`)
-const { store, emitter } = require(`../redux`)
+const { store, emitter, readState } = require(`../redux`)
 const queryUtil = require(`../query`)
 const appDataUtil = require(`../utils/app-data`)
 const WorkerPool = require(`../utils/worker/pool`)
 const { structureWebpackErrors } = require(`../utils/webpack-error-utils`)
+const pageDataUtil = require(`../utils/page-data`)
 const {
   waitUntilAllJobsComplete: waitUntilAllJobsV2Complete,
 } = require(`../utils/jobs-manager`)
@@ -48,6 +49,8 @@ const waitUntilAllJobsComplete = () => {
 
 module.exports = async function build(program: BuildArgs) {
   const publicDir = path.join(program.directory, `public`)
+  const incrementalBuild =
+    process.env.GATSBY_INCREMENTAL_BUILD === `true` || false
   initTracer(program.openTracingConfigFile)
   const buildActivity = report.phantomActivity(`build`)
   buildActivity.start()
@@ -68,7 +71,7 @@ module.exports = async function build(program: BuildArgs) {
   const {
     processPageQueries,
     processStaticQueries,
-  } = queryUtil.getInitialQueryProcessors({
+  } = await queryUtil.getInitialQueryProcessors({
     parentSpan: buildSpan,
   })
 
@@ -100,7 +103,7 @@ module.exports = async function build(program: BuildArgs) {
   const webpackCompilationHash = stats.hash
   if (
     webpackCompilationHash !== store.getState().webpackCompilationHash ||
-    !appDataUtil.exists(publicDir)
+    (!incrementalBuild && !appDataUtil.exists(publicDir))
   ) {
     store.dispatch({
       type: `SET_WEBPACK_COMPILATION_HASH`,
@@ -136,15 +139,14 @@ module.exports = async function build(program: BuildArgs) {
   require(`../redux/actions`).boundActionCreators.setProgramStatus(
     `BOOTSTRAP_QUERY_RUNNING_FINISHED`
   )
-
-  await db.saveState()
-
   await waitUntilAllJobsComplete()
 
-  // we need to save it again to make sure our latest state has been saved
-  await db.saveState()
-
-  const pagePaths = [...store.getState().pages.keys()]
+  activity = report.activityTimer(`Building static HTML for pages`, {
+    parentSpan: buildSpan,
+  })
+  const pagePaths = incrementalBuild
+    ? await pageDataUtil.getNewPageKeys(store.getState(), readState())
+    : [...store.getState().pages.keys()]
   activity = report.createProgress(
     `Building static HTML for pages`,
     pagePaths.length,
@@ -184,18 +186,75 @@ module.exports = async function build(program: BuildArgs) {
   }
   activity.done()
 
+  let deletedPageKeys = []
+  if (incrementalBuild) {
+    activity = report.activityTimer(`Delete previous page data`)
+    activity.start()
+    deletedPageKeys = await pageDataUtil.removePreviousPageData(
+      program.directory,
+      store.getState(),
+      readState()
+    )
+    activity.end()
+  }
+
   await apiRunnerNode(`onPostBuild`, {
     graphql: graphqlRunner,
     parentSpan: buildSpan,
   })
 
   // Make sure we saved the latest state so we have all jobs cached
+  activity = report.activityTimer(`Update cache for next build`, {
+    parentSpan: buildSpan,
+  })
+  activity.start()
   await db.saveState()
+  activity.end()
 
   report.info(`Done building in ${process.uptime()} sec`)
 
   buildSpan.finish()
   await stopTracer()
   workerPool.end()
   buildActivity.end()
+  if (incrementalBuild && process.argv.indexOf(`--log-pages`) > -1) {
+    if (pagePaths.length) {
+      report.info(
+        `Incremental build pages:\n${pagePaths.map(
+          path => `Updated page: ${path}\n`
+        )}`.replace(/,/g, ``)
+      )
+    }
+    if (deletedPageKeys.length) {
+      report.info(
+        `Incremental build deleted pages:\n${deletedPageKeys.map(
+          path => `Deleted page: ${path}\n`
+        )}`.replace(/,/g, ``)
+      )
+    }
+  }
+
+  if (incrementalBuild && process.argv.indexOf(`--write-to-file`) > -1) {
+    const createdFilesPath = path.resolve(
+      `${program.directory}/.cache`,
+      `newPages.txt`
+    )
+    const deletedFilesPath = path.resolve(
+      `${program.directory}/.cache`,
+      `deletedPages.txt`
+    )
+
+    if (pagePaths.length) {
+      fs.writeFileSync(createdFilesPath, `${pagePaths.join(`\n`)}\n`, `utf8`)
+      report.info(`newPages.txt created`)
+    }
+    if (deletedPageKeys.length) {
+      fs.writeFileSync(
+        deletedFilesPath,
+        `${deletedPageKeys.join(`\n`)}\n`,
+        `utf8`
+      )
+      report.info(`deletedPages.txt created`)
+    }
+  }
 }

packages/gatsby/src/query/index.js

@@ -3,11 +3,12 @@
 const _ = require(`lodash`)
 const Queue = require(`better-queue`)
 // const convertHrtime = require(`convert-hrtime`)
-const { store, emitter } = require(`../redux`)
+const { store, emitter, readState } = require(`../redux`)
 const { boundActionCreators } = require(`../redux/actions`)
 const report = require(`gatsby-cli/lib/reporter`)
 const queryQueue = require(`./queue`)
 const GraphQLRunner = require(`./graphql-runner`)
+const pageDataUtil = require(`../utils/page-data`)
 
 const seenIdsWithoutDataDependencies = new Set()
 let queuedDirtyActions = []
@@ -148,10 +149,21 @@ const calcInitialDirtyQueryIds = state => {
 /**
  * groups queryIds by whether they are static or page queries.
  */
-const groupQueryIds = queryIds => {
+const groupQueryIds = async queryIds => {
+  const incrementalBuild =
+    process.env.GATSBY_INCREMENTAL_BUILD === `true` || false
   const grouped = _.groupBy(queryIds, p =>
     p.slice(0, 4) === `sq--` ? `static` : `page`
   )
+
+  if (incrementalBuild) {
+    const newPageKeys = await pageDataUtil.getNewPageKeys(
+      store.getState(),
+      readState()
+    )
+    grouped.page = newPageKeys
+  }
+
   return {
     staticQueryIds: grouped.static || [],
     pageQueryIds: grouped.page || [],
@@ -219,10 +231,10 @@ const processPageQueries = async (queryIds, { state, activity }) => {
   )
 }
 
-const getInitialQueryProcessors = ({ parentSpan } = {}) => {
+const getInitialQueryProcessors = async ({ parentSpan } = {}) => {
   const state = store.getState()
   const queryIds = calcInitialDirtyQueryIds(state)
-  const { staticQueryIds, pageQueryIds } = groupQueryIds(queryIds)
+  const { staticQueryIds, pageQueryIds } = await groupQueryIds(queryIds)
 
   const queryjobsCount =
     _.filter(pageQueryIds.map(id => state.pages.get(id))).length +
@@ -256,7 +268,7 @@ const initialProcessQueries = async ({ parentSpan } = {}) => {
     pageQueryIds,
     processPageQueries,
     processStaticQueries,
-  } = getInitialQueryProcessors({ parentSpan })
+  } = await getInitialQueryProcessors({ parentSpan })
 
   await processStaticQueries()
   await processPageQueries()
@@ -290,10 +302,10 @@ let listenerQueue
  * Run any dirty queries. See `calcQueries` for what constitutes a
  * dirty query
  */
-const runQueuedQueries = () => {
+const runQueuedQueries = async () => {
   if (listenerQueue) {
     const state = store.getState()
-    const { staticQueryIds, pageQueryIds } = groupQueryIds(
+    const { staticQueryIds, pageQueryIds } = await groupQueryIds(
       calcDirtyQueryIds(state)
     )
     const pages = _.filter(pageQueryIds.map(id => state.pages.get(id)))
@@ -317,7 +329,7 @@ const runQueuedQueries = () => {
  * For what constitutes a dirty query, see `calcQueries`
  */
 
-const startListeningToDevelopQueue = () => {
+const startListeningToDevelopQueue = async () => {
   // We use a queue to process batches of queries so that they are
   // processed consecutively
   let graphqlRunner = null
@@ -345,7 +357,7 @@ const startListeningToDevelopQueue = () => {
     report.pendingActivity({ id: `query-running` })
   })
 
-  emitter.on(`API_RUNNING_QUEUE_EMPTY`, runQueuedQueries)
+  emitter.on(`API_RUNNING_QUEUE_EMPTY`, await runQueuedQueries)
   ;[
     `DELETE_CACHE`,
     `CREATE_NODE`,

packages/gatsby/src/redux/__tests__/index.js

@@ -64,6 +64,6 @@ describe(`redux db`, () => {
     expect(data.components).not.toEqual(initialComponentsState)
 
     // yuck - loki and redux will have different shape of redux state (nodes and nodesByType)
-    expect(_.omit(data, [`nodes`, `nodesByType`])).toMatchSnapshot()
+    expect(_.omit(data, [`nodes`, `nodesByType`, `pages`])).toMatchSnapshot()
   })
 })

packages/gatsby/src/redux/index.ts

@@ -60,14 +60,20 @@ export const configureStore = (initialState: IReduxState): Store<IReduxState> =>
     applyMiddleware(thunk, multi)
   )
 
-export const store = configureStore(readState())
+const initialState = readState()
+// Page data is not required to be in the initial redux store.
+// This will enable us to make a comparison of the cached state and new state.
+// Allowing us to add and delete pages.
+initialState.pages = new Map()
+export const store = configureStore(initialState) // Persist state.
 
 // Persist state.
 export const saveState = (): void => {
   const state = store.getState()
   const pickedState = _.pick(state, [
     `nodes`,
     `status`,
+    `pages`,
     `componentDataDependencies`,
     `components`,
     `jobsV2`,

packages/gatsby/src/redux/types.ts

@@ -11,6 +11,7 @@ export interface IReduxState {
     developMiddleware: any
     proxy: any
   }
+  pages?: any
 }
 
 export enum ProgramStatus {