cache backlink records #33444

gatsbyjs · Oct 25, 2021 · 3fa6041 · 3fa6041
1 parent abd640c
commit 3fa6041
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 19 deletions.
diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js
@@ -20,6 +20,15 @@ jest.mock(`gatsby-source-filesystem`, () => {
   }
 })
 
+function makeCache() {
+  const store = new Map()
+  return {
+    get: async id => store.get(id),
+    set: async (key, value) => store.set(key, value),
+    store,
+  }
+}
+
 const normalize = require(`../normalize`)
 const downloadFileSpy = jest.spyOn(normalize, `downloadFile`)
 
@@ -75,6 +84,7 @@ describe(`gatsby-source-drupal`, () => {
     store,
     getNode: id => nodes[id],
     getNodes,
+    cache: makeCache(),
   }
 
   beforeAll(async () => {

diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js
@@ -12,6 +12,8 @@ const { setOptions, getOptions } = require(`./plugin-options`)
 
 const { nodeFromData, downloadFile, isFileNode } = require(`./normalize`)
 const {
+  initRefsLookups,
+  storeRefsLookups,
   handleReferences,
   handleWebhookUpdate,
   handleDeletedNode,
@@ -62,9 +64,6 @@ async function worker([url, options]) {
   const response = await got(url, {
     agent,
     cache: false,
-    timeout: {
-      request: 15000,
-    },
     // request: http2wrapper.auto,
     // http2: true,
     ...options,
@@ -153,6 +152,8 @@ exports.sourceNodes = async (
   } = pluginOptions
   const { createNode, setPluginStatus, touchNode } = actions
 
+  await initRefsLookups({ cache, getNode })
+
   // Update the concurrency limit from the plugin options
   requestQueue.concurrency = concurrentAPIRequests
 
@@ -205,6 +206,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
         }
 
         changesActivity.end()
+        await storeRefsLookups({ cache })
         return
       }
 
@@ -235,6 +237,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
       return
     }
     changesActivity.end()
+    await storeRefsLookups({ cache })
     return
   }
 
@@ -256,7 +259,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
 
     // lastFetched isn't set so do a full rebuild.
     if (!lastFetched) {
-      setPluginStatus({ lastFetched: Math.floor(new Date().getTime() / 1000) })
+      setPluginStatus({ lastFetched: new Date().getTime() })
       requireFullRebuild = true
     } else {
       const drupalFetchIncrementalActivity = reporter.activityTimer(
@@ -267,14 +270,9 @@ ${JSON.stringify(webhookBody, null, 4)}`
       drupalFetchIncrementalActivity.start()
 
       try {
-        console.time(`drupal: gatsby-fastbuilds/sync`)
         // Hit fastbuilds endpoint with the lastFetched date.
         const res = await requestQueue.push([
-          urlJoin(
-            baseUrl,
-            `gatsby-fastbuilds/sync/`,
-            Math.floor(lastFetched).toString()
-          ),
+          urlJoin(baseUrl, `gatsby-fastbuilds/sync/`, lastFetched.toString()),
           {
             username: basicAuth.username,
             password: basicAuth.password,
@@ -285,8 +283,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
           },
         ])
 
-        console.timeEnd(`drupal: gatsby-fastbuilds/sync`)
-
         // Fastbuilds returns a -1 if:
         // - the timestamp has expired
         // - if old fastbuild logs were purged
@@ -299,7 +295,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
         } else {
           // Touch nodes so they are not garbage collected by Gatsby.
           if (initialSourcing) {
-            console.time(`drupal: touchNode`)
             const touchNodesSpan = tracer.startSpan(`sourceNodes.touchNodes`, {
               childOf: fastBuildsSpan,
             })
@@ -311,7 +306,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
                 touchNode(node)
               }
             })
-            console.timeEnd(`drupal: touchNode`)
             touchNodesSpan.setTag(`sourceNodes.touchNodes.count`, touchCount)
             touchNodesSpan.finish()
           }
@@ -327,7 +321,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
           )
 
           // Process sync data from Drupal.
-          console.time(`drupal: process synced data`)
           const nodesToSync = res.body.entities
           for (const nodeSyncData of nodesToSync) {
             if (nodeSyncData.action === `delete`) {
@@ -366,7 +359,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
               }
             }
           }
-          console.timeEnd(`drupal: process synced data`)
 
           createNodesSpan.finish()
           setPluginStatus({ lastFetched: res.body.timestamp })
@@ -376,6 +368,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
 
         drupalFetchIncrementalActivity.end()
         fastBuildsSpan.finish()
+        await storeRefsLookups({ cache })
         return
       }
 
@@ -386,6 +379,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
       initialSourcing = false
 
       if (!requireFullRebuild) {
+        await storeRefsLookups({ cache })
         return
       }
     }
@@ -649,6 +643,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
   initialSourcing = false
 
   createNodesSpan.finish()
+  await storeRefsLookups({ cache, getNodes })
   return
 }
 

diff --git a/packages/gatsby-source-drupal/src/utils.js b/packages/gatsby-source-drupal/src/utils.js
@@ -9,8 +9,38 @@ const {
 
 const { getOptions } = require(`./plugin-options`)
 
-const backRefsNamesLookup = new Map()
-const referencedNodesLookup = new Map()
+let backRefsNamesLookup = new Map()
+let referencedNodesLookup = new Map()
+
+const initRefsLookups = async ({ cache }) => {
+  const backRefsNamesLookupStr = await cache.get(`backRefsNamesLookup`)
+  const referencedNodesLookupStr = await cache.get(`referencedNodesLookup`)
+
+  if (backRefsNamesLookupStr) {
+    backRefsNamesLookup = new Map(JSON.parse(backRefsNamesLookupStr))
+  }
+
+  if (referencedNodesLookupStr) {
+    referencedNodesLookup = new Map(JSON.parse(referencedNodesLookupStr))
+  }
+}
+
+exports.initRefsLookups = initRefsLookups
+
+const storeRefsLookups = async ({ cache }) => {
+  await Promise.all([
+    cache.set(
+      `backRefsNamesLookup`,
+      JSON.stringify(Array.from(backRefsNamesLookup.entries()))
+    ),
+    cache.set(
+      `referencedNodesLookup`,
+      JSON.stringify(Array.from(referencedNodesLookup.entries()))
+    ),
+  ])
+}
+
+exports.storeRefsLookups = storeRefsLookups
 
 const handleReferences = (
   node,
@@ -334,7 +364,7 @@ ${JSON.stringify(nodeToUpdate, null, 4)}
     node.internal.contentDigest = createContentDigest(node)
     createNode(node)
     reporter.log(
-      `Updated Gatsby node — id: ${node.id} type: ${node.internal.type}`
+      `Updated Gatsby node: id: ${node.id} — type: ${node.internal.type}`
     )
   }
 }