diff --git a/packages/gatsby-source-drupal/package.json b/packages/gatsby-source-drupal/package.json index e60da6eb75265..0f616a3d2759f 100644 --- a/packages/gatsby-source-drupal/package.json +++ b/packages/gatsby-source-drupal/package.json @@ -8,12 +8,13 @@ }, "dependencies": { "@babel/runtime": "^7.14.0", - "got": "^11.8.2", "agentkeepalive": "^4.1.1", - "fastq": "^1.11.0", "bluebird": "^3.7.2", "body-parser": "^1.19.0", + "fastq": "^1.11.0", "gatsby-source-filesystem": "^3.8.0-next.1", + "got": "^11.8.2", + "http2-wrapper": "^2.0.5", "lodash": "^4.17.21", "tiny-async-pool": "^1.2.0", "url-join": "^4.0.1" @@ -24,8 +25,15 @@ "babel-preset-gatsby-package": "^1.8.0-next.1", "cross-env": "^7.0.3" }, + "engines": { + "node": ">=12.13.0" + }, "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-drupal#readme", - "keywords": ["gatsby", "gatsby-plugin", "gatsby-source-plugin"], + "keywords": [ + "gatsby", + "gatsby-plugin", + "gatsby-source-plugin" + ], "license": "MIT", "peerDependencies": { "gatsby": "^3.0.0-next.0" @@ -39,8 +47,5 @@ "build": "babel src --out-dir . --ignore \"**/__tests__\"", "prepare": "cross-env NODE_ENV=production npm run build", "watch": "babel -w src --out-dir . --ignore \"**/__tests__\"" - }, - "engines": { - "node": ">=12.13.0" } } diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index a75fec25c3c23..7311ed87d50cb 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -2,6 +2,7 @@ const got = require(`got`) const _ = require(`lodash`) const urlJoin = require(`url-join`) import HttpAgent from "agentkeepalive" +const http2wrapper = require(`http2-wrapper`) const { HttpsAgent } = HttpAgent @@ -18,10 +19,17 @@ const { handleReferences, handleWebhookUpdate } = require(`./utils`) const agent = { http: new HttpAgent(), https: new HttpsAgent(), + http2: new http2wrapper.Agent(), } async function worker([url, options]) { - return got(url, { agent, ...options }) + return got(url, { + agent, + cache: false, + request: http2wrapper.auto, + http2: true, + ...options, + }) } const requestQueue = require(`fastq`).promise(worker, 20) @@ -170,10 +178,13 @@ exports.sourceNodes = async ( headers, searchParams: params, responseType: `json`, - cache, }, ]) + // Fastbuilds returns a -1 if: + // - the timestamp has expired + // - if old fastbuild logs were purged + // - it's been a really long time since you synced so you just do a full fetch. if (res.body.status === -1) { // The incremental data is expired or this is the first fetch. reporter.info(`Unable to pull incremental data changes from Drupal`) @@ -267,11 +278,11 @@ exports.sourceNodes = async ( headers, searchParams: params, responseType: `json`, - cache, }, ]) allData = await Promise.all( _.map(res.body.links, async (url, type) => { + const dataArray = [] if (disallowedLinkTypes.includes(type)) return if (!url) return if (!type) return @@ -281,7 +292,7 @@ exports.sourceNodes = async ( entityType => entityType === type ) - const getNext = async (url, data = []) => { + const getNext = async url => { if (typeof url === `object`) { // url can be string or object containing href field url = url.href @@ -315,37 +326,33 @@ exports.sourceNodes = async ( password: basicAuth.password, headers, responseType: `json`, - cache, }, ]) } catch (error) { if (error.response && error.response.statusCode == 405) { // The endpoint doesn't support the GET method, so just skip it. - return [] + return } else { console.error(`Failed to fetch ${url}`, error.message) console.log(error) throw error } } - data = data.concat(d.body.data) + dataArray.push(...d.body.data) // Add support for includes. Includes allow entity data to be expanded // based on relationships. The expanded data is exposed as `included` // in the JSON API response. // See https://www.drupal.org/docs/8/modules/jsonapi/includes if (d.body.included) { - data = data.concat(d.body.included) + dataArray.push(...d.body.included) } if (d.body.links && d.body.links.next) { - data = await getNext(d.body.links.next, data) + await getNext(d.body.links.next) } - - return data } - let data = [] if (isTranslatable === false) { - data = await getNext(url) + await getNext(url) } else { for (let i = 0; i < languageConfig.enabledLanguages.length; i++) { let currentLanguage = languageConfig.enabledLanguages[i] @@ -368,13 +375,13 @@ exports.sourceNodes = async ( ) const dataForLanguage = await getNext(joinedUrl) - data = data.concat(dataForLanguage) + dataArray.push(...dataForLanguage) } } const result = { type, - data, + data: dataArray, } // eslint-disable-next-line consistent-return diff --git a/yarn.lock b/yarn.lock index 34a06482dadeb..b2944de42c525 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14446,6 +14446,14 @@ http2-wrapper@^1.0.0-beta.5.2: quick-lru "^5.1.1" resolve-alpn "^1.0.0" +http2-wrapper@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-2.0.5.tgz#d4464509df69b6f82125b6cb337dbb80101f406c" + integrity sha512-W8+pfYl0iQ27NjvhDrbuQKaMBjBAWIZRHdKvmctV2c8a/naI7SsZgU3e04lCYrgxqnJ2sNPsBBrVI8kBeE/Pig== + dependencies: + quick-lru "^5.1.1" + resolve-alpn "^1.1.1" + httperrors@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/httperrors/-/httperrors-2.0.1.tgz#02febcaec8d9d6a9e1ae3773915b9fdaa2204672" @@ -24367,6 +24375,11 @@ resolve-alpn@^1.0.0: resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.0.0.tgz#745ad60b3d6aff4b4a48e01b8c0bdc70959e0e8c" integrity sha512-rTuiIEqFmGxne4IovivKSDzld2lWW9QCjqv80SYjPgf+gS35eaCAjaP54CCwGAwBtnCsvNLYtqxe1Nw+i6JEmA== +resolve-alpn@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.1.2.tgz#30b60cfbb0c0b8dc897940fe13fe255afcdd4d28" + integrity sha512-8OyfzhAtA32LVUsJSke3auIyINcwdh5l3cvYKdKO0nvsYSKuiLfTM5i78PJswFPT8y6cPW+L1v6/hE95chcpDA== + resolve-cwd@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-2.0.0.tgz#00a9f7387556e27038eae232caa372a6a59b665a"