feat(gatsby, gatsby-source-contentful): add public action to disable stale node checks

docs/docs/node-creation.md

@@ -65,6 +65,33 @@ When a `source-nodes` plugin runs again, it generally recreates nodes (which aut
 
 Any nodes that aren't touched by the end of the `source-nodes` phase, are deleted. This is performed via a diff between the `nodesTouched` and `nodes` Redux namespaces, in [source-nodes.ts](https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby/src/utils/source-nodes.ts)
 
+### Opting out of stale node deletion
+
+Stale node deletion is a very expensive process because all nodes in the data store need to be iterated on to check if they're stale or not. Iterating on nodes to check for staleness also requires loading that entire node into memory, so all nodes created by a source plugin are loaded into memory to check for their staleness, even if they're otherwise not needed in memory.
+
+Source plugins can skip this expensive step by calling the `enableStatefulSourceNodes` action.
+This will stop Gatsby from checking for stale nodes created by the source plugin that called the action.
+This is a major performance improvement for medium and large sites and those sites will need less total memory to build.
+
+When enabling stateful sourcing plugin authors need to be sure their plugins properly handle deleting nodes when they need to be deleted. Since Gatsby is no longer checking for node staleness, data which should no longer exist could stick around.
+`enableStatefulSourceNodes` should only be enabled for source plugins that can fully support all delete operations in their data source.
+
+Note that if `enableStatefulSourceNodes` is supported by the user's `gatsby` version, the action should be called every time `sourceNodes` runs.
+
+Example:
+
+```js
+import { hasFeature } from "gatsby-plugin-utils"
+
+exports.sourceNodes = ({ actions }) => {
+  if (hasFeature(`stateful-source-nodes`)) {
+    actions.enableStatefulSourceNodes()
+  } else {
+    // fallback to old behavior where all nodes are iterated on and touchNode is called.
+  }
+}
+```
+
 ## Changing a node's fields
 
 From a site developer's point of view, nodes are immutable. In the sense that if you change a node object, those changes will not be seen by other parts of Gatsby. To make a change to a node, it must be persisted to Redux via an action.

packages/gatsby-source-contentful/src/__tests__/gatsby-node.js

@@ -87,6 +87,7 @@ describe(`gatsby-node`, () => {
       currentNodeMap.delete(node.id)
     }),
     touchNode: jest.fn(),
+    enableStatefulSourceNodes: jest.fn(),
   }
   const schema = {
     buildObjectType: jest.fn(() => {

packages/gatsby-source-contentful/src/normalize.js

@@ -5,7 +5,8 @@ import { getGatsbyVersion } from "gatsby-core-utils"
 import { lt, prerelease } from "semver"
 
 const typePrefix = `Contentful`
-const makeTypeName = type => _.upperFirst(_.camelCase(`${typePrefix} ${type}`))
+export const makeTypeName = type =>
+  _.upperFirst(_.camelCase(`${typePrefix} ${type}`))
 
 const GATSBY_VERSION_MANIFEST_V2 = `4.3.0`
 const gatsbyVersion =

packages/gatsby-source-contentful/src/source-nodes.js

@@ -1,4 +1,5 @@
 // @ts-check
+import { hasFeature } from "gatsby-plugin-utils/has-feature"
 import isOnline from "is-online"
 import _ from "lodash"
 
@@ -11,6 +12,7 @@ import {
   createAssetNodes,
   createNodesForContentType,
   makeId,
+  makeTypeName,
 } from "./normalize"
 import { createPluginConfig } from "./plugin-options"
 import { CODES } from "./report"
@@ -40,7 +42,7 @@ const CONTENT_DIGEST_COUNTER_SEPARATOR = `_COUNT_`
  * or the fallback field or the default field.
  */
 
-let isFirstSource = true
+let isFirstSourceNodesCallOfCurrentNodeProcess = true
 export async function sourceNodes(
   {
     actions,
@@ -55,14 +57,26 @@ export async function sourceNodes(
   },
   pluginOptions
 ) {
-  const { createNode, touchNode, deleteNode, unstable_createNodeManifest } =
-    actions
+  const hasStatefulSourceNodes = hasFeature(`stateful-source-nodes`)
+  const needToTouchNodes = !hasStatefulSourceNodes
+
+  const {
+    createNode,
+    touchNode,
+    deleteNode,
+    unstable_createNodeManifest,
+    enableStatefulSourceNodes,
+  } = actions
+
   const online = await isOnline()
 
+  if (hasStatefulSourceNodes) {
+    enableStatefulSourceNodes()
+  }
   // Gatsby only checks if a node has been touched on the first sourcing.
   // As iterating and touching nodes can grow quite expensive on larger sites with
   // 1000s of nodes, we'll skip doing this on subsequent sources.
-  if (isFirstSource) {
+  else if (isFirstSourceNodesCallOfCurrentNodeProcess && needToTouchNodes) {
     getNodes().forEach(node => {
       if (node.internal.owner !== `gatsby-source-contentful`) {
         return
@@ -73,9 +87,10 @@ export async function sourceNodes(
         touchNode(getNode(node.fields.localFile))
       }
     })
-    isFirstSource = false
   }
 
+  isFirstSourceNodesCallOfCurrentNodeProcess = false
+
   if (
     !online &&
     process.env.GATSBY_CONTENTFUL_OFFLINE === `true` &&

packages/gatsby/index.d.ts

@@ -21,6 +21,7 @@ export type AvailableFeatures =
   | "image-cdn"
   | "graphql-typegen"
   | "content-file-path"
+  | "stateful-source-nodes"
 
 export {
   Link,
@@ -433,6 +434,11 @@ export interface GatsbyNode<
     calllback: PluginCallback<void>
   ): void | Promise<void>
 
+  /**
+   * Marks the source plugin that called this function as stateful. Gatsby will not check for stale nodes for any plugin that calls this.
+   */
+  enableStatefulSourceNodes?(this: void, plugin?: ActionPlugin)
+
   /**
    * Called when a new node is created. Plugins wishing to extend or
    * transform nodes created by other plugins should implement this API.

packages/gatsby/scripts/__tests__/api.js

@@ -35,6 +35,7 @@ it("generates the expected api output", done => {
           "graphql-typegen",
           "content-file-path",
           "slices",
+          "stateful-source-nodes",
         ],
         "node": Object {
           "createPages": Object {},

packages/gatsby/scripts/output-api-file.js

@@ -41,7 +41,7 @@ async function outputFile() {
   }, {})
 
   /** @type {Array<import("../index").AvailableFeatures>} */
-  output.features = ["image-cdn", "graphql-typegen", "content-file-path", "slices"];
+  output.features = ["image-cdn", "graphql-typegen", "content-file-path", "slices", "stateful-source-nodes"];
 
   return fs.writeFile(
     path.resolve(OUTPUT_FILE_NAME),

packages/gatsby/src/datastore/__tests__/nodes.js

@@ -339,4 +339,103 @@ describe(`nodes db tests`, () => {
     })
     expect(getNodes()).toHaveLength(0)
   })
+
+  it(`records the node type owner when a node is created`, async () => {
+    // creating a node
+    store.dispatch(
+      actions.createNode(
+        {
+          id: `1`,
+          internal: {
+            type: `OwnerOneTestTypeOne`,
+            contentDigest: `ok`,
+          },
+        },
+        {
+          name: `test-owner-1`,
+        }
+      )
+    )
+    // and creating a second node in the same type
+    store.dispatch(
+      actions.createNode(
+        {
+          id: `2`,
+          internal: {
+            type: `OwnerOneTestTypeOne`,
+            contentDigest: `ok`,
+          },
+        },
+        {
+          name: `test-owner-1`,
+        }
+      )
+    )
+
+    // and a third node of a different type but same plugin
+    store.dispatch(
+      actions.createNode(
+        {
+          id: `3`,
+          internal: {
+            type: `OwnerOneTestTypeTwo`,
+            contentDigest: `ok`,
+          },
+        },
+        {
+          name: `test-owner-1`,
+        }
+      )
+    )
+
+    // fourth node by a different plugin
+    store.dispatch(
+      actions.createNode(
+        {
+          id: `4`,
+          internal: {
+            type: `OwnerTwoTestTypeThree`,
+            contentDigest: `ok`,
+          },
+        },
+        {
+          name: `test-owner-2`,
+        }
+      )
+    )
+
+    // fifth node by second plugin but the node is deleted. Deleted nodes still have type owners
+    const nodeFive = {
+      id: `5`,
+      internal: {
+        type: `OwnerTwoTestTypeFour`,
+        contentDigest: `ok`,
+      },
+    }
+    store.dispatch(
+      actions.createNode(nodeFive, {
+        name: `test-owner-2`,
+      })
+    )
+    store.dispatch(
+      actions.deleteNode(nodeFive, {
+        name: `test-owner-2`,
+      })
+    )
+    expect(getNode(`5`)).toBeUndefined()
+
+    const state = store.getState()
+
+    const ownerOne = state.typeOwners.pluginsToTypes.get(`test-owner-1`)
+    expect(ownerOne).toBeTruthy()
+    expect(ownerOne.has(`OwnerOneTestTypeOne`)).toBeTrue()
+    expect(ownerOne.has(`OwnerOneTestTypeTwo`)).toBeTrue()
+    expect(ownerOne.has(`OwnerTwoTestTypeThree`)).toBeFalse()
+
+    const ownerTwo = state.typeOwners.pluginsToTypes.get(`test-owner-2`)
+    expect(ownerTwo).toBeTruthy()
+    expect(ownerTwo.has(`OwnerOneTestTypeTwo`)).toBeFalse()
+    expect(ownerTwo.has(`OwnerTwoTestTypeThree`)).toBeTrue()
+    expect(ownerTwo.has(`OwnerTwoTestTypeFour`)).toBeTrue()
+  })
 })

packages/gatsby/src/query/__tests__/data-tracking.js

@@ -70,6 +70,13 @@ jest.mock(`gatsby-cli/lib/reporter`, () => {
         end: jest.fn(),
       }
     },
+    createProgress: () => {
+      return {
+        start: jest.fn(),
+        tick: jest.fn(),
+        end: jest.fn(),
+      }
+    },
     phantomActivity: () => {
       return {
         start: jest.fn(),

packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap

@@ -123,13 +123,18 @@ Object {
   },
   "slices": Map {},
   "slicesByTemplate": Map {},
+  "statefulSourcePlugins": Set {},
   "staticQueriesByTemplate": Map {},
   "staticQueryComponents": Map {},
   "status": Object {
     "LAST_NODE_COUNTER": 0,
     "PLUGINS_HASH": "",
     "plugins": Object {},
   },
+  "typeOwners": Object {
+    "pluginsToTypes": Map {},
+    "typesToPlugins": Map {},
+  },
   "webpackCompilationHash": "",
 }
 `;

packages/gatsby/src/redux/__tests__/nodes.ts

@@ -4,8 +4,7 @@ import { nodesReducer } from "../reducers/nodes"
 import { IGatsbyNode } from "../types"
 import { nodesTouchedReducer } from "../reducers/nodes-touched"
 
-const dispatch = jest.fn()
-
+const dispatch = jest.spyOn(store, `dispatch`)
 type MapObject = Record<string, IGatsbyNode>
 
 const fromMapToObject = (map: Map<string, any>): MapObject => {
@@ -18,8 +17,8 @@ const fromMapToObject = (map: Map<string, any>): MapObject => {
 
 describe(`Create and update nodes`, (): void => {
   beforeEach((): void => {
-    dispatch.mockClear()
     store.dispatch({ type: `DELETE_CACHE` })
+    dispatch.mockClear()
   })
 
   it(`allows creating nodes`, (): void => {
@@ -80,7 +79,7 @@ describe(`Create and update nodes`, (): void => {
         children: [],
         parent: `test`,
         internal: {
-          contentDigest: `hasdfljds`,
+          contentDigest: `hasdfljds2`,
           type: `Test`,
         },
         pickle: false,