From b33f5feca334a5f95b2838f7a393b52128b05c86 Mon Sep 17 00:00:00 2001 From: Vladimir Razuvaev Date: Wed, 23 Jun 2021 03:05:48 +0700 Subject: [PATCH] feat(gatsby-source-drupal): Disable caching + add http/2 agent (#32012) (#32038) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(gatsby-source-drupal): Force revalidation on Drupal API requests Talking to Gatsby/Drupal users — most set long max-age for the Drupal cache to keep their edge http cache as fresh as possible as Drupal can directly purge its edge http cache. But this has the unfortunate side-effect with the recent http client change in https://github.com/gatsbyjs/gatsby/pull/31514 that API calls aren't revalidated. Meaning that a user could change some content in Drupal and not see those updates in their Gatsby site until the Drupal cache expires in the Gatsby cache. This PR removes the `cache-control` header from Drupal API responses so that we only can use `etag` for caching which forces revalidation on every request. * Remove debugging line * Add http/2 agent * Disable cache — it's slower than refetch with revalidation * push don't concat * consistent return * Fix statusCode check for got * Actually this was correct... * Actually use http/2 (cherry picked from commit 113e43e19b1086e74c55f7092e4a66b7e9fe7c95) # Conflicts: # packages/gatsby-source-drupal/package.json Co-authored-by: Kyle Mathews --- packages/gatsby-source-drupal/package.json | 17 ++++++--- .../gatsby-source-drupal/src/gatsby-node.js | 37 +++++++++++-------- yarn.lock | 13 +++++++ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/packages/gatsby-source-drupal/package.json b/packages/gatsby-source-drupal/package.json index e60da6eb75265..0f616a3d2759f 100644 --- a/packages/gatsby-source-drupal/package.json +++ b/packages/gatsby-source-drupal/package.json @@ -8,12 +8,13 @@ }, "dependencies": { "@babel/runtime": "^7.14.0", - "got": "^11.8.2", "agentkeepalive": "^4.1.1", - "fastq": "^1.11.0", "bluebird": "^3.7.2", "body-parser": "^1.19.0", + "fastq": "^1.11.0", "gatsby-source-filesystem": "^3.8.0-next.1", + "got": "^11.8.2", + "http2-wrapper": "^2.0.5", "lodash": "^4.17.21", "tiny-async-pool": "^1.2.0", "url-join": "^4.0.1" @@ -24,8 +25,15 @@ "babel-preset-gatsby-package": "^1.8.0-next.1", "cross-env": "^7.0.3" }, + "engines": { + "node": ">=12.13.0" + }, "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-drupal#readme", - "keywords": ["gatsby", "gatsby-plugin", "gatsby-source-plugin"], + "keywords": [ + "gatsby", + "gatsby-plugin", + "gatsby-source-plugin" + ], "license": "MIT", "peerDependencies": { "gatsby": "^3.0.0-next.0" @@ -39,8 +47,5 @@ "build": "babel src --out-dir . --ignore \"**/__tests__\"", "prepare": "cross-env NODE_ENV=production npm run build", "watch": "babel -w src --out-dir . --ignore \"**/__tests__\"" - }, - "engines": { - "node": ">=12.13.0" } } diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index a75fec25c3c23..7311ed87d50cb 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -2,6 +2,7 @@ const got = require(`got`) const _ = require(`lodash`) const urlJoin = require(`url-join`) import HttpAgent from "agentkeepalive" +const http2wrapper = require(`http2-wrapper`) const { HttpsAgent } = HttpAgent @@ -18,10 +19,17 @@ const { handleReferences, handleWebhookUpdate } = require(`./utils`) const agent = { http: new HttpAgent(), https: new HttpsAgent(), + http2: new http2wrapper.Agent(), } async function worker([url, options]) { - return got(url, { agent, ...options }) + return got(url, { + agent, + cache: false, + request: http2wrapper.auto, + http2: true, + ...options, + }) } const requestQueue = require(`fastq`).promise(worker, 20) @@ -170,10 +178,13 @@ exports.sourceNodes = async ( headers, searchParams: params, responseType: `json`, - cache, }, ]) + // Fastbuilds returns a -1 if: + // - the timestamp has expired + // - if old fastbuild logs were purged + // - it's been a really long time since you synced so you just do a full fetch. if (res.body.status === -1) { // The incremental data is expired or this is the first fetch. reporter.info(`Unable to pull incremental data changes from Drupal`) @@ -267,11 +278,11 @@ exports.sourceNodes = async ( headers, searchParams: params, responseType: `json`, - cache, }, ]) allData = await Promise.all( _.map(res.body.links, async (url, type) => { + const dataArray = [] if (disallowedLinkTypes.includes(type)) return if (!url) return if (!type) return @@ -281,7 +292,7 @@ exports.sourceNodes = async ( entityType => entityType === type ) - const getNext = async (url, data = []) => { + const getNext = async url => { if (typeof url === `object`) { // url can be string or object containing href field url = url.href @@ -315,37 +326,33 @@ exports.sourceNodes = async ( password: basicAuth.password, headers, responseType: `json`, - cache, }, ]) } catch (error) { if (error.response && error.response.statusCode == 405) { // The endpoint doesn't support the GET method, so just skip it. - return [] + return } else { console.error(`Failed to fetch ${url}`, error.message) console.log(error) throw error } } - data = data.concat(d.body.data) + dataArray.push(...d.body.data) // Add support for includes. Includes allow entity data to be expanded // based on relationships. The expanded data is exposed as `included` // in the JSON API response. // See https://www.drupal.org/docs/8/modules/jsonapi/includes if (d.body.included) { - data = data.concat(d.body.included) + dataArray.push(...d.body.included) } if (d.body.links && d.body.links.next) { - data = await getNext(d.body.links.next, data) + await getNext(d.body.links.next) } - - return data } - let data = [] if (isTranslatable === false) { - data = await getNext(url) + await getNext(url) } else { for (let i = 0; i < languageConfig.enabledLanguages.length; i++) { let currentLanguage = languageConfig.enabledLanguages[i] @@ -368,13 +375,13 @@ exports.sourceNodes = async ( ) const dataForLanguage = await getNext(joinedUrl) - data = data.concat(dataForLanguage) + dataArray.push(...dataForLanguage) } } const result = { type, - data, + data: dataArray, } // eslint-disable-next-line consistent-return diff --git a/yarn.lock b/yarn.lock index 34a06482dadeb..b2944de42c525 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14446,6 +14446,14 @@ http2-wrapper@^1.0.0-beta.5.2: quick-lru "^5.1.1" resolve-alpn "^1.0.0" +http2-wrapper@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-2.0.5.tgz#d4464509df69b6f82125b6cb337dbb80101f406c" + integrity sha512-W8+pfYl0iQ27NjvhDrbuQKaMBjBAWIZRHdKvmctV2c8a/naI7SsZgU3e04lCYrgxqnJ2sNPsBBrVI8kBeE/Pig== + dependencies: + quick-lru "^5.1.1" + resolve-alpn "^1.1.1" + httperrors@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/httperrors/-/httperrors-2.0.1.tgz#02febcaec8d9d6a9e1ae3773915b9fdaa2204672" @@ -24367,6 +24375,11 @@ resolve-alpn@^1.0.0: resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.0.0.tgz#745ad60b3d6aff4b4a48e01b8c0bdc70959e0e8c" integrity sha512-rTuiIEqFmGxne4IovivKSDzld2lWW9QCjqv80SYjPgf+gS35eaCAjaP54CCwGAwBtnCsvNLYtqxe1Nw+i6JEmA== +resolve-alpn@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.1.2.tgz#30b60cfbb0c0b8dc897940fe13fe255afcdd4d28" + integrity sha512-8OyfzhAtA32LVUsJSke3auIyINcwdh5l3cvYKdKO0nvsYSKuiLfTM5i78PJswFPT8y6cPW+L1v6/hE95chcpDA== + resolve-cwd@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-2.0.0.tgz#00a9f7387556e27038eae232caa372a6a59b665a"