Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(gatsby-source-wordpress): opt out of stale node checks #37920

Merged
merged 3 commits into from
Apr 13, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { Step } from "./../utils/run-steps"
import store from "~/store"
import { getGatsbyApi } from "~/utils/get-gatsby-api"
import { getPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "~/utils/gatsby-features"

const persistPreviouslyCachedImages: Step = async (): Promise<void> => {
const { helpers, pluginOptions } = getGatsbyApi()
Expand All @@ -11,9 +12,11 @@ const persistPreviouslyCachedImages: Step = async (): Promise<void> => {
`${pluginOptions.schema.typePrefix}MediaItem`
)

// and touch them so they aren't garbage collected.
// we will remove them as needed when receiving DELETE events from WP
mediaItemNodes.forEach(node => helpers.actions.touchNode(node))
if (needToTouchNodes) {
// and if needed touch them so they aren't garbage collected.
// we will remove them as needed when receiving DELETE events from WP
mediaItemNodes.forEach(node => helpers.actions.touchNode(node))
}

const imageNodeMetaByUrl = await getPersistentCache({
key: `image-node-meta-by-url`,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { createNodeWithSideEffects } from "./create-nodes"
import fetchReferencedMediaItemsAndCreateNodes from "../fetch-nodes/fetch-referenced-media-items"
import { CREATED_NODE_IDS } from "~/constants"
import { getPersistentCache, setPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "../../../utils/gatsby-features"

const fetchAndCreateNonNodeRootFields = async () => {
const state = store.getState()
Expand Down Expand Up @@ -69,20 +70,22 @@ const fetchAndCreateNonNodeRootFields = async () => {
referencedMediaItemNodeIds: newMediaItemIds,
})

const previouslyCachedNodeIds = await getPersistentCache({
key: CREATED_NODE_IDS,
})

const createdNodeIds = [
...new Set([
...(previouslyCachedNodeIds || []),
...referencedMediaItemNodeIdsArray,
]),
]

// save the node id's so we can touch them on the next build
// so that we don't have to refetch all nodes
await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
if (needToTouchNodes) {
const previouslyCachedNodeIds = await getPersistentCache({
key: CREATED_NODE_IDS,
})

const createdNodeIds = [
...new Set([
...(previouslyCachedNodeIds || []),
...referencedMediaItemNodeIdsArray,
]),
]

// save the node id's so we can touch them on the next build
// so that we don't have to refetch all nodes
await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
}

store.dispatch.logger.stopActivityTimer({
typeName: `MediaItems`,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
setPersistentCache,
} from "~/utils/cache"
import { buildTypeName } from "../../create-schema-customization/helpers"
import { needToTouchNodes } from "../../../utils/gatsby-features"

/**
* fetchWPGQLContentNodes
Expand Down Expand Up @@ -227,7 +228,9 @@ export const fetchAndCreateAllNodes = async () => {
})
}

// save the node id's so we can touch them on the next build
// so that we don't have to refetch all nodes
await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
if (needToTouchNodes) {
// save the node id's so we can touch them on the next build
// so that we don't have to refetch all nodes
await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds })
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,14 @@ import store from "~/store"
import fetchAndCreateNonNodeRootFields from "./create-nodes/fetch-and-create-non-node-root-fields"
import { allowFileDownloaderProgressBarToClear } from "./create-nodes/create-remote-file-node/progress-bar-promise"
import { sourcePreviews } from "~/steps/preview"
import { hasStatefulSourceNodes } from "~/utils/gatsby-features"

const sourceNodes: Step = async helpers => {
const { cache, webhookBody, refetchAll } = helpers
const { cache, webhookBody, refetchAll, actions } = helpers

if (hasStatefulSourceNodes) {
actions.enableStatefulSourceNodes()
pieh marked this conversation as resolved.
Show resolved Hide resolved
}

// fetch non-node root fields such as settings.
// For now, we're refetching them on every build
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,13 @@ import { fetchAndRunWpActions } from "./wp-actions"
import { formatLogMessage } from "~/utils/format-log-message"
import { getGatsbyApi } from "~/utils/get-gatsby-api"
import { getPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "../../../utils/gatsby-features"

export const touchValidNodes = async () => {
if (!needToTouchNodes) {
return
}

const { helpers } = getGatsbyApi()
const { actions } = helpers

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,15 @@ import { fetchGraphql } from "~/utils/fetch-graphql"
import { getQueryInfoBySingleFieldName } from "../../helpers"
import { CREATED_NODE_IDS } from "~/constants"
import { setPersistentCache, getPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "../../../../utils/gatsby-features"

const wpActionDELETE = async ({
helpers,
// cachedNodeIds,
wpAction,
}) => {
const wpActionDELETE = async ({ helpers, wpAction }) => {
const { reporter, actions, getNode } = helpers

try {
const cachedNodeIds = await getPersistentCache({ key: CREATED_NODE_IDS })
const cachedNodeIds = needToTouchNodes
? await getPersistentCache({ key: CREATED_NODE_IDS })
: null

// get the node ID from the WPGQL id
const nodeId = wpAction.referencedNodeGlobalRelayID
Expand Down Expand Up @@ -60,7 +59,7 @@ const wpActionDELETE = async ({
wpStore: store,
})) || {}

if (additionalNodeIds && additionalNodeIds.length) {
if (needToTouchNodes && additionalNodeIds && additionalNodeIds.length) {
additionalNodeIds.forEach(id => cachedNodeIds.push(id))
}
}
Expand All @@ -81,12 +80,12 @@ const wpActionDELETE = async ({
reporter.log(``)
}

// Remove this from cached node id's so we don't try to touch it
const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)
if (needToTouchNodes) {
// Remove this from cached node id's so we don't try to touch it
const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)

await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })

// return validNodeIds
await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
}
} catch (e) {
Object.entries(wpAction).forEach(([key, value]) => {
reporter.warn(`${key} ${JSON.stringify(value)}`)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
} from "~/steps/create-schema-customization/helpers"
import { processNode } from "~/steps/source-nodes/create-nodes/process-node"
import { getPersistentCache, setPersistentCache } from "~/utils/cache"
import { needToTouchNodes } from "../../../../utils/gatsby-features"

export const fetchAndCreateSingleNode = async ({
singleName,
Expand Down Expand Up @@ -170,7 +171,7 @@ export const createSingleNode = async ({

const { typeInfo } = getQueryInfoBySingleFieldName(singleName)

if (!cachedNodeIds) {
if (!cachedNodeIds && needToTouchNodes) {
cachedNodeIds = await getPersistentCache({ key: CREATED_NODE_IDS })
}

Expand Down Expand Up @@ -259,12 +260,11 @@ export const createSingleNode = async ({

if (remoteNode) {
actions.createNode(remoteNode)
}

if (remoteNode && needToTouchNodes) {
cachedNodeIds.push(remoteNode.id)

if (additionalNodeIds && additionalNodeIds.length) {
additionalNodeIds.forEach(id => cachedNodeIds.push(id))
}
additionalNodeIds?.forEach(id => cachedNodeIds.push(id))

await setPersistentCache({ key: CREATED_NODE_IDS, value: cachedNodeIds })
}
Expand Down Expand Up @@ -304,13 +304,17 @@ const wpActionUPDATE = async ({ helpers, wpAction }) => {
const existingNode = await getNode(nodeId)

if (wpAction.referencedNodeStatus !== `publish`) {
// if the post status isn't publish anymore, we need to remove the node
// by removing it from cached nodes so it's garbage collected by Gatsby
const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)
if (needToTouchNodes) {
// if the post status isn't publish anymore, we need to remove the node
// by removing it from cached nodes so it's garbage collected by Gatsby
const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId)

await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds })
}

if (existingNode) {
// calling touchNode here ensures owners is populated before we delete.
// In some cases calling delete node without touching the node first throws an error. this is a bug in Gatsby that needs to be fixed
TylerBarnes marked this conversation as resolved.
Show resolved Hide resolved
await actions.touchNode(existingNode)
await actions.deleteNode(existingNode)
reportUpdate({ setAction: `DELETE` })
Expand Down
4 changes: 4 additions & 0 deletions packages/gatsby-source-wordpress/src/utils/gatsby-features.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import { hasFeature } from "gatsby-plugin-utils"

export const hasStatefulSourceNodes = hasFeature(`stateful-source-nodes`)
export const needToTouchNodes = !hasStatefulSourceNodes