From 76f979c68f8d8b2c5204bc102e94f48ed5b2e4eb Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 12 Jan 2023 15:31:54 +0100 Subject: [PATCH] fix(gatsby-source-contentful): maintain back reference map between runs (#37442) * test(gatsby-source-contentful): add test case * fix(gatsby-source-contentful): maintain back reference map between runs --- .../src/__fixtures__/preserve-back-links.js | 339 ++++++++++++++++++ .../src/__tests__/gatsby-node.js | 48 +++ .../src/__tests__/normalize.js | 54 ++- .../gatsby-source-contentful/src/normalize.js | 66 +++- .../src/source-nodes.js | 11 +- 5 files changed, 491 insertions(+), 27 deletions(-) create mode 100644 packages/gatsby-source-contentful/src/__fixtures__/preserve-back-links.js diff --git a/packages/gatsby-source-contentful/src/__fixtures__/preserve-back-links.js b/packages/gatsby-source-contentful/src/__fixtures__/preserve-back-links.js new file mode 100644 index 0000000000000..ca11460e154e4 --- /dev/null +++ b/packages/gatsby-source-contentful/src/__fixtures__/preserve-back-links.js @@ -0,0 +1,339 @@ +exports.contentTypeItems = () => [ + { + sys: { + space: { + sys: { + type: `Link`, + linkType: `Space`, + id: `8itggr1zebzx`, + }, + }, + id: `blogPost`, + type: `ContentType`, + createdAt: `2023-01-11T14:52:56.250Z`, + updatedAt: `2023-01-11T14:54:56.940Z`, + environment: { + sys: { + id: `master`, + type: `Link`, + linkType: `Environment`, + }, + }, + revision: 4, + }, + displayField: `title`, + name: `Blog Post`, + description: ``, + fields: [ + { + id: `title`, + name: `Title`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `slug`, + name: `Slug`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `body`, + name: `Body`, + type: `Text`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `category`, + name: `Category`, + type: `Link`, + localized: false, + required: false, + disabled: false, + omitted: false, + linkType: `Entry`, + validations: [ + { + linkContentType: [`blogCategory`], + }, + ], + }, + ], + }, + { + sys: { + space: { + sys: { + type: `Link`, + linkType: `Space`, + id: `8itggr1zebzx`, + }, + }, + id: `blogCategory`, + type: `ContentType`, + createdAt: `2023-01-11T14:54:22.680Z`, + updatedAt: `2023-01-11T14:54:22.680Z`, + environment: { + sys: { + id: `master`, + type: `Link`, + linkType: `Environment`, + }, + }, + revision: 1, + }, + displayField: `title`, + name: `Blog Category`, + description: ``, + fields: [ + { + id: `title`, + name: `Title`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `slug`, + name: `Slug`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + ], + }, +] + +exports.initialSync = () => { + return { + currentSyncData: { + entries: [ + { + metadata: { + tags: [], + }, + sys: { + space: { + sys: { + type: `Link`, + linkType: `Space`, + id: `8itggr1zebzx`, + }, + }, + id: `3jXBlUgXmubzPI3I6d9hLr`, + type: `Entry`, + createdAt: `2023-01-11T14:56:37.418Z`, + updatedAt: `2023-01-11T15:04:37.640Z`, + environment: { + sys: { + id: `master`, + type: `Link`, + linkType: `Environment`, + }, + }, + revision: 3, + contentType: { + sys: { + type: `Link`, + linkType: `ContentType`, + id: `blogCategory`, + }, + }, + }, + fields: { + title: { + "en-US": `CMS`, + }, + slug: { + "en-US": `cms`, + }, + }, + }, + { + metadata: { + tags: [], + }, + sys: { + space: { + sys: { + type: `Link`, + linkType: `Space`, + id: `8itggr1zebzx`, + }, + }, + id: `3oTFYoNKoVZcp8svbn8P2z`, + type: `Entry`, + createdAt: `2023-01-11T14:56:42.655Z`, + updatedAt: `2023-01-11T14:56:42.655Z`, + environment: { + sys: { + id: `master`, + type: `Link`, + linkType: `Environment`, + }, + }, + revision: 1, + contentType: { + sys: { + type: `Link`, + linkType: `ContentType`, + id: `blogPost`, + }, + }, + }, + fields: { + title: { + "en-US": `Hello World`, + }, + slug: { + "en-US": `hello-world`, + }, + body: { + "en-US": `Lorem ipsum`, + }, + category: { + "en-US": { + sys: { + type: `Link`, + linkType: `Entry`, + id: `3jXBlUgXmubzPI3I6d9hLr`, + }, + }, + }, + }, + }, + ], + assets: [], + deletedEntries: [], + deletedAssets: [], + nextSyncToken: `dDFSNcK6bMO7woHDuMK7A8O_KWQDPkhAwpF6w7ovw49fQjrDj2gKH0xvwofCkMKDJcKgBMKCYcK9wr3DoVozwqEUC8OwWlVJBBt-F8K0BMKTP8OAwr8Xw6bCkcO2w6MpwqBmVX7CmsOwM3DDvWZvw5Q`, + }, + tagItems: [], + defaultLocale: `en-US`, + locales: [ + { + code: `en-US`, + name: `English (United States)`, + default: true, + fallbackCode: null, + sys: { + id: `2jpGtQkqT01zpSIqC9UQOS`, + type: `Locale`, + version: 1, + }, + }, + ], + space: { + sys: { + type: `Space`, + id: `8itggr1zebzx`, + }, + name: `test`, + locales: [ + { + code: `en-US`, + default: true, + name: `English (United States)`, + fallbackCode: null, + }, + ], + }, + } +} + +exports.editJustEntryWithBackLinks = () => { + return { + currentSyncData: { + entries: [ + { + metadata: { + tags: [], + }, + sys: { + space: { + sys: { + type: `Link`, + linkType: `Space`, + id: `8itggr1zebzx`, + }, + }, + id: `3jXBlUgXmubzPI3I6d9hLr`, + type: `Entry`, + createdAt: `2023-01-11T14:56:37.418Z`, + updatedAt: `2023-01-11T15:06:18.306Z`, + environment: { + sys: { + id: `master`, + type: `Link`, + linkType: `Environment`, + }, + }, + revision: 4, + contentType: { + sys: { + type: `Link`, + linkType: `ContentType`, + id: `blogCategory`, + }, + }, + }, + fields: { + title: { + "en-US": `CMS edit #1`, + }, + slug: { + "en-US": `cms`, + }, + }, + }, + ], + assets: [], + deletedEntries: [], + deletedAssets: [], + nextSyncToken: `dDFSNcK6bMO7woHDuMK7A8O_KWQDPkhAwpF6w7ovw49fQjrDj2gKH0xvQMODwpLDkMK3Oj9Jw6jDkSoBMkc4woTCtMOFwoTDisKlT8O1w4AaKsOjasK1wrVSwrU3YsKVE8KPVMKyw4_CmVpwPsOew4IVwoA`, + }, + tagItems: [], + defaultLocale: `en-US`, + locales: [ + { + code: `en-US`, + name: `English (United States)`, + default: true, + fallbackCode: null, + sys: { + id: `2jpGtQkqT01zpSIqC9UQOS`, + type: `Locale`, + version: 1, + }, + }, + ], + space: { + sys: { + type: `Space`, + id: `8itggr1zebzx`, + }, + name: `test`, + locales: [ + { + code: `en-US`, + default: true, + name: `English (United States)`, + fallbackCode: null, + }, + ], + }, + } +} diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 381cb5db9dcb5..d8ad217174f65 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -14,6 +14,7 @@ import richTextFixture from "../__fixtures__/rich-text-data" import restrictedContentTypeFixture from "../__fixtures__/restricted-content-type" import unpublishedFieldDelivery from "../__fixtures__/unpublished-fields-delivery" import unpublishedFieldPreview from "../__fixtures__/unpublished-fields-preview" +import preserveBackLinks from "../__fixtures__/preserve-back-links" jest.mock(`../fetch`) jest.mock(`gatsby-core-utils`, () => { @@ -534,6 +535,9 @@ describe(`gatsby-node`, () => { Array [ "contentful-content-types-testSpaceId-master", ], + Array [ + "contentful-foreign-reference-map-state-testSpaceId-master", + ], ] `) @@ -546,6 +550,7 @@ describe(`gatsby-node`, () => { expect(cache.set.mock.calls.map(v => v[0])).toMatchInlineSnapshot(` Array [ "contentful-content-types-testSpaceId-master", + "contentful-foreign-reference-map-state-testSpaceId-master", ] `) expect(actions.createNode).toHaveBeenCalledTimes(32) @@ -1307,4 +1312,47 @@ describe(`gatsby-node`, () => { }) ) }) + + it(`should preserve back reference when referencing entry wasn't touched`, async () => { + // @ts-ignore + fetchContentTypes.mockImplementation(preserveBackLinks.contentTypeItems) + fetchContent + // @ts-ignore + .mockImplementationOnce(preserveBackLinks.initialSync) + .mockImplementationOnce(preserveBackLinks.editJustEntryWithBackLinks) + + let blogPostNodes + let blogCategoryNodes + await simulateGatsbyBuild() + + blogPostNodes = getNodes().filter( + node => node.internal.type === `ContentfulBlogPost` + ) + blogCategoryNodes = getNodes().filter( + node => node.internal.type === `ContentfulBlogCategory` + ) + + expect(blogPostNodes.length).toEqual(1) + expect(blogCategoryNodes.length).toEqual(1) + expect(blogCategoryNodes[0][`blog post___NODE`]).toEqual([ + blogPostNodes[0].id, + ]) + expect(blogCategoryNodes[0][`title`]).toEqual(`CMS`) + + await simulateGatsbyBuild() + + blogPostNodes = getNodes().filter( + node => node.internal.type === `ContentfulBlogPost` + ) + blogCategoryNodes = getNodes().filter( + node => node.internal.type === `ContentfulBlogCategory` + ) + + expect(blogPostNodes.length).toEqual(1) + expect(blogCategoryNodes.length).toEqual(1) + expect(blogCategoryNodes[0][`blog post___NODE`]).toEqual([ + blogPostNodes[0].id, + ]) + expect(blogCategoryNodes[0][`title`]).toEqual(`CMS edit #1`) + }) }) diff --git a/packages/gatsby-source-contentful/src/__tests__/normalize.js b/packages/gatsby-source-contentful/src/__tests__/normalize.js index 0f6ca5884f8ae..2e1f2be440e17 100644 --- a/packages/gatsby-source-contentful/src/__tests__/normalize.js +++ b/packages/gatsby-source-contentful/src/__tests__/normalize.js @@ -101,15 +101,17 @@ describe(`generic`, () => { entryList, }) - const foreignReferenceMap = buildForeignReferenceMap({ + const foreignReferenceMapState = buildForeignReferenceMap({ contentTypeItems, entryList, resolvable, defaultLocale, space, useNameForId: true, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) - const referenceKeys = Object.keys(foreignReferenceMap) + const referenceKeys = Object.keys(foreignReferenceMapState.backLinks) const expectedReferenceKeys = [ `2Y8LhXLnYAYqKCGEWG4EKI___Asset`, `3wtvPBbBjiMKqKKga8I2Cu___Asset`, @@ -129,7 +131,7 @@ describe(`generic`, () => { expect(referenceKeys).toHaveLength(expectedReferenceKeys.length) expect(referenceKeys).toEqual(expect.arrayContaining(expectedReferenceKeys)) - Object.keys(foreignReferenceMap).forEach(referenceId => { + Object.keys(foreignReferenceMapState.backLinks).forEach(referenceId => { expect(resolvable).toContain(referenceId) let expectedLength = 1 @@ -139,7 +141,9 @@ describe(`generic`, () => { if (referenceId === `7LAnCobuuWYSqks6wAwY2a___Entry`) { expectedLength = 3 } - expect(foreignReferenceMap[referenceId]).toHaveLength(expectedLength) + expect(foreignReferenceMapState.backLinks[referenceId]).toHaveLength( + expectedLength + ) }) }) }) @@ -156,22 +160,26 @@ describe(`Process contentful data (by name)`, () => { entryList, }) - const foreignReferenceMap = buildForeignReferenceMap({ + const foreignReferenceMapState = buildForeignReferenceMap({ contentTypeItems, entryList, resolvable, defaultLocale, space, useNameForId: true, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) - expect(foreignReferenceMap[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0].name).toBe( - `product___NODE` - ) + expect( + foreignReferenceMapState.backLinks[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0] + .name + ).toBe(`product___NODE`) - expect(foreignReferenceMap[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0].name).toBe( - `brand___NODE` - ) + expect( + foreignReferenceMapState.backLinks[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0] + .name + ).toBe(`brand___NODE`) }) it(`creates nodes for each entry`, () => { @@ -192,6 +200,8 @@ describe(`Process contentful data (by name)`, () => { defaultLocale, space, useNameForId: true, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) const createNode = jest.fn() @@ -291,6 +301,8 @@ describe(`Process existing mutated nodes in warm build`, () => { defaultLocale, space, useNameForId: true, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) const createNode = jest.fn() @@ -377,22 +389,26 @@ describe(`Process contentful data (by id)`, () => { assets: currentSyncData.assets, entryList, }) - const foreignReferenceMap = buildForeignReferenceMap({ + const foreignReferenceMapState = buildForeignReferenceMap({ contentTypeItems, entryList, resolvable, defaultLocale, space, useNameForId: false, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) - expect(foreignReferenceMap[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0].name).toBe( - `2pqfxujwe8qsykum0u6w8m___NODE` - ) + expect( + foreignReferenceMapState.backLinks[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0] + .name + ).toBe(`2pqfxujwe8qsykum0u6w8m___NODE`) - expect(foreignReferenceMap[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0].name).toBe( - `sfztzbsum8coewygeuyes___NODE` - ) + expect( + foreignReferenceMapState.backLinks[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0] + .name + ).toBe(`sfztzbsum8coewygeuyes___NODE`) }) it(`creates nodes for each entry`, () => { @@ -411,6 +427,8 @@ describe(`Process contentful data (by id)`, () => { defaultLocale, space, useNameForId: false, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) const createNode = jest.fn() diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 23e655b04e27d..1b0c0e8f32014 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -101,6 +101,28 @@ export const buildResolvableSet = ({ return resolvable } +function cleanupReferencesFromEntry(foreignReferenceMapState, entry) { + const { links, backLinks } = foreignReferenceMapState + const entryId = entry.sys.id + + const entryLinks = links[entryId] + if (entryLinks) { + entryLinks.forEach(link => { + const backLinksForLink = backLinks[link] + if (backLinksForLink) { + const newBackLinks = backLinksForLink.filter(({ id }) => id !== entryId) + if (newBackLinks.lenth > 0) { + backLinks[link] = newBackLinks + } else { + delete backLinks[link] + } + } + }) + } + + delete links[entryId] +} + export const buildForeignReferenceMap = ({ contentTypeItems, entryList, @@ -108,8 +130,21 @@ export const buildForeignReferenceMap = ({ defaultLocale, space, useNameForId, + previousForeignReferenceMapState, + deletedEntries, }) => { - const foreignReferenceMap = {} + const foreignReferenceMapState = previousForeignReferenceMapState || { + links: {}, + backLinks: {}, + } + + const { links, backLinks } = foreignReferenceMapState + + for (const deletedEntry of deletedEntries) { + // remove stored entries from entry that is being deleted + cleanupReferencesFromEntry(foreignReferenceMapState, deletedEntry) + } + contentTypeItems.forEach((contentTypeItem, i) => { // Establish identifier for content type // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, @@ -122,6 +157,9 @@ export const buildForeignReferenceMap = ({ } entryList[i].forEach(entryItem => { + // clear links added in previous runs for given entry, as we will recreate them anyway + cleanupReferencesFromEntry(foreignReferenceMapState, entryItem) + const entryItemFields = entryItem.fields Object.keys(entryItemFields).forEach(entryItemFieldKey => { if (entryItemFields[entryItemFieldKey]) { @@ -143,15 +181,21 @@ export const buildForeignReferenceMap = ({ return } - if (!foreignReferenceMap[key]) { - foreignReferenceMap[key] = [] + if (!backLinks[key]) { + backLinks[key] = [] } - foreignReferenceMap[key].push({ + backLinks[key].push({ name: `${contentTypeItemId}___NODE`, id: entryItem.sys.id, spaceId: space.sys.id, type: entryItem.sys.type, }) + + if (!links[entryItem.sys.id]) { + links[entryItem.sys.id] = [] + } + + links[entryItem.sys.id].push(key) }) } } else if ( @@ -166,22 +210,28 @@ export const buildForeignReferenceMap = ({ return } - if (!foreignReferenceMap[key]) { - foreignReferenceMap[key] = [] + if (!backLinks[key]) { + backLinks[key] = [] } - foreignReferenceMap[key].push({ + backLinks[key].push({ name: `${contentTypeItemId}___NODE`, id: entryItem.sys.id, spaceId: space.sys.id, type: entryItem.sys.type, }) + + if (!links[entryItem.sys.id]) { + links[entryItem.sys.id] = [] + } + + links[entryItem.sys.id].push(key) } } }) }) }) - return foreignReferenceMap + return foreignReferenceMapState } function prepareTextNode(id, node, key, text) { diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index 5b9927de5cf82..87ca7a8bd982a 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -117,6 +117,7 @@ export async function sourceNodes( const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` + const CACHE_FOREIGN_REFERENCE_MAP_STATE = `contentful-foreign-reference-map-state-${sourceId}` /* * Subsequent calls of Contentfuls sync API return only changed data. @@ -240,16 +241,24 @@ export async function sourceNodes( assets, }) + const previousForeignReferenceMapState = await cache.get( + CACHE_FOREIGN_REFERENCE_MAP_STATE + ) // Build foreign reference map before starting to insert any nodes - const foreignReferenceMap = buildForeignReferenceMap({ + const foreignReferenceMapState = buildForeignReferenceMap({ contentTypeItems, entryList, resolvable, defaultLocale, space, useNameForId: pluginConfig.get(`useNameForId`), + previousForeignReferenceMapState, + deletedEntries: currentSyncData?.deletedEntries, }) + await cache.set(CACHE_FOREIGN_REFERENCE_MAP_STATE, foreignReferenceMapState) + const foreignReferenceMap = foreignReferenceMapState.backLinks + reporter.verbose(`Resolving Contentful references`) const newOrUpdatedEntries = new Set()