Skip to content

Commit

Permalink
feat(gatsby-source-wordpress): opt out of stale node checks (#37920)
Browse files Browse the repository at this point in the history
* opt out of stale node checks

* update error message

---------

Co-authored-by: Michal Piechowiak <misiek.piechowiak@gmail.com>
  • Loading branch information
TylerBarnes and pieh authored Apr 13, 2023
1 parent e258553 commit 0e9feeb
Show file tree
Hide file tree
Showing 8 changed files with 69 additions and 42 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { Step } from "./../utils/run-steps"
import store from "~/store"
import { getGatsbyApi } from "~/utils/get-gatsby-api"
import { getPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "~/utils/gatsby-features"

const persistPreviouslyCachedImages: Step = async (): Promise<void> => {
const { helpers, pluginOptions } = getGatsbyApi()
Expand All @@ -11,9 +12,11 @@ const persistPreviouslyCachedImages: Step = async (): Promise<void> => {
`${pluginOptions.schema.typePrefix}MediaItem`
)

// and touch them so they aren't garbage collected.
// we will remove them as needed when receiving DELETE events from WP
mediaItemNodes.forEach(node => helpers.actions.touchNode(node))
if (needToTouchNodes) {
// and if needed touch them so they aren't garbage collected.
// we will remove them as needed when receiving DELETE events from WP
mediaItemNodes.forEach(node => helpers.actions.touchNode(node))
}

const imageNodeMetaByUrl = await getPersistentCache({
key: `image-node-meta-by-url`,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { createNodeWithSideEffects } from "./create-nodes"
import fetchReferencedMediaItemsAndCreateNodes from "../fetch-nodes/fetch-referenced-media-items"
import { CREATED_NODE_IDS } from "~/constants"
import { getPersistentCache, setPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "../../../utils/gatsby-features"

const fetchAndCreateNonNodeRootFields = async () => {
const state = store.getState()
Expand Down Expand Up @@ -69,20 +70,22 @@ const fetchAndCreateNonNodeRootFields = async () => {
referencedMediaItemNodeIds: newMediaItemIds,
})

const previouslyCachedNodeIds = await getPersistentCache({
key: CREATED_NODE_IDS,
})

const createdNodeIds = [
...new Set([
...(previouslyCachedNodeIds || []),
...referencedMediaItemNodeIdsArray,
]),
]

// save the node id's so we can touch them on the next build
// so that we don't have to refetch all nodes
await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
if (needToTouchNodes) {
const previouslyCachedNodeIds = await getPersistentCache({
key: CREATED_NODE_IDS,
})

const createdNodeIds = [
...new Set([
...(previouslyCachedNodeIds || []),
...referencedMediaItemNodeIdsArray,
]),
]

// save the node id's so we can touch them on the next build
// so that we don't have to refetch all nodes
await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
}

store.dispatch.logger.stopActivityTimer({
typeName: `MediaItems`,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
setPersistentCache,
} from "~/utils/cache"
import { buildTypeName } from "../../create-schema-customization/helpers"
import { needToTouchNodes } from "../../../utils/gatsby-features"

/**
* fetchWPGQLContentNodes
Expand Down Expand Up @@ -227,7 +228,9 @@ export const fetchAndCreateAllNodes = async () => {
})
}

// save the node id's so we can touch them on the next build
// so that we don't have to refetch all nodes
await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
if (needToTouchNodes) {
// save the node id's so we can touch them on the next build
// so that we don't have to refetch all nodes
await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,14 @@ import store from "~/store"
import fetchAndCreateNonNodeRootFields from "./create-nodes/fetch-and-create-non-node-root-fields"
import { allowFileDownloaderProgressBarToClear } from "./create-nodes/create-remote-file-node/progress-bar-promise"
import { sourcePreviews } from "~/steps/preview"
import { hasStatefulSourceNodes } from "~/utils/gatsby-features"

const sourceNodes: Step = async helpers => {
const { cache, webhookBody, refetchAll } = helpers
const { cache, webhookBody, refetchAll, actions } = helpers

if (hasStatefulSourceNodes) {
actions.enableStatefulSourceNodes()
}

// fetch non-node root fields such as settings.
// For now, we're refetching them on every build
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,13 @@ import { fetchAndRunWpActions } from "./wp-actions"
import { formatLogMessage } from "~/utils/format-log-message"
import { getGatsbyApi } from "~/utils/get-gatsby-api"
import { getPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "../../../utils/gatsby-features"

export const touchValidNodes = async () => {
if (!needToTouchNodes) {
return
}

const { helpers } = getGatsbyApi()
const { actions } = helpers

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,15 @@ import { fetchGraphql } from "~/utils/fetch-graphql"
import { getQueryInfoBySingleFieldName } from "../../helpers"
import { CREATED_NODE_IDS } from "~/constants"
import { setPersistentCache, getPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "../../../../utils/gatsby-features"

const wpActionDELETE = async ({
helpers,
// cachedNodeIds,
wpAction,
}) => {
const wpActionDELETE = async ({ helpers, wpAction }) => {
const { reporter, actions, getNode } = helpers

try {
const cachedNodeIds = await getPersistentCache({ key: CREATED_NODE_IDS })
const cachedNodeIds = needToTouchNodes
? await getPersistentCache({ key: CREATED_NODE_IDS })
: null

// get the node ID from the WPGQL id
const nodeId = wpAction.referencedNodeGlobalRelayID
Expand Down Expand Up @@ -60,7 +59,7 @@ const wpActionDELETE = async ({
wpStore: store,
})) || {}

if (additionalNodeIds && additionalNodeIds.length) {
if (needToTouchNodes && additionalNodeIds && additionalNodeIds.length) {
additionalNodeIds.forEach(id => cachedNodeIds.push(id))
}
}
Expand All @@ -81,12 +80,12 @@ const wpActionDELETE = async ({
reporter.log(``)
}

// Remove this from cached node id's so we don't try to touch it
const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)
if (needToTouchNodes) {
// Remove this from cached node id's so we don't try to touch it
const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)

await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })

// return validNodeIds
await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
}
} catch (e) {
Object.entries(wpAction).forEach(([key, value]) => {
reporter.warn(`${key} ${JSON.stringify(value)}`)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
} from "~/steps/create-schema-customization/helpers"
import { processNode } from "~/steps/source-nodes/create-nodes/process-node"
import { getPersistentCache, setPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "../../../../utils/gatsby-features"

export const fetchAndCreateSingleNode = async ({
singleName,
Expand Down Expand Up @@ -170,7 +171,7 @@ export const createSingleNode = async ({

const { typeInfo } = getQueryInfoBySingleFieldName(singleName)

if (!cachedNodeIds) {
if (!cachedNodeIds && needToTouchNodes) {
cachedNodeIds = await getPersistentCache({ key: CREATED_NODE_IDS })
}

Expand Down Expand Up @@ -259,12 +260,11 @@ export const createSingleNode = async ({

if (remoteNode) {
actions.createNode(remoteNode)
}

if (remoteNode && needToTouchNodes) {
cachedNodeIds.push(remoteNode.id)

if (additionalNodeIds && additionalNodeIds.length) {
additionalNodeIds.forEach(id => cachedNodeIds.push(id))
}
additionalNodeIds?.forEach(id => cachedNodeIds.push(id))

await setPersistentCache({ key: CREATED_NODE_IDS, value: cachedNodeIds })
}
Expand Down Expand Up @@ -304,13 +304,18 @@ const wpActionUPDATE = async ({ helpers, wpAction }) => {
const existingNode = await getNode(nodeId)

if (wpAction.referencedNodeStatus !== `publish`) {
// if the post status isn't publish anymore, we need to remove the node
// by removing it from cached nodes so it's garbage collected by Gatsby
const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)
if (needToTouchNodes) {
// if the post status isn't publish anymore, we need to remove the node
// by removing it from cached nodes so it's garbage collected by Gatsby
const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)

await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
}

if (existingNode) {
// calling touchNode here ensures owners is populated before we delete.
// In some cases calling delete node without touching the node first throws an error. this is a bug in Gatsby that has been fixed but this code remains to be backwards compatible with earlier versions that have the bug still.
// TODO remove in the next major version
await actions.touchNode(existingNode)
await actions.deleteNode(existingNode)
reportUpdate({ setAction: `DELETE` })
Expand Down
4 changes: 4 additions & 0 deletions packages/gatsby-source-wordpress/src/utils/gatsby-features.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import { hasFeature } from "gatsby-plugin-utils"

export const hasStatefulSourceNodes = hasFeature(`stateful-source-nodes`)
export const needToTouchNodes = !hasStatefulSourceNodes

0 comments on commit 0e9feeb

Please sign in to comment.