From 024e1121bf8f5e6946182c68a5424ec6fd128e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Fichot?= Date: Wed, 9 Aug 2017 22:42:03 +0200 Subject: [PATCH 01/15] Update README.md Unecessary per_page mention in readme --- packages/gatsby-source-wordpress/README.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/packages/gatsby-source-wordpress/README.md b/packages/gatsby-source-wordpress/README.md index 860082e20861e..5bb074269d076 100644 --- a/packages/gatsby-source-wordpress/README.md +++ b/packages/gatsby-source-wordpress/README.md @@ -95,23 +95,6 @@ The Authentication on Wordpress.com is not supported yet. This means that you wo ] ``` - -### How to raise the default and maximum number of posts to be returned in result set from REST API - -To modify the `per_page` `default` and `maximum` arguments add the following to your functions.php - -```php - add_filter( 'rest_endpoints', function( $endpoints ){ - if ( ! isset( $endpoints['/wp/v2/posts'] ) ) { - return $endpoints; - } - $endpoints['/wp/v2/posts'][0]['args']['per_page']['default'] = 200; // defaults to 10 - $endpoints['/wp/v2/posts'][0]['args']['per_page']['maximum'] = 200; // defaults to 100 - return $endpoints; - }); -``` - - ## How to query : GraphQL You can query nodes created from Wordpress using GraphQL like the following: From 64ddedd5a2b278c9922188353e8e7af5a8787863 Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Sun, 6 Aug 2017 11:23:51 +0200 Subject: [PATCH 02/15] Simplify `httpExceptionHandler` with es6 --- .../src/gatsby-node.js | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index beb289d7344cb..4b8be3e61755e 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -170,35 +170,33 @@ async function axiosHelper(url) { /** * Handles HTTP Exceptions (axios) - * - * @param {any} e + * + * @param {any} e */ -function httpExceptionHandler(e) { +function httpExceptionHandler (e) { + const { status, statusText, data: { message } } = e.response console.log( colorized.out( - `The server response was "${e.response.status} ${e.response.statusText}"`, + `The server response was "${status} ${statusText}"`, colorized.color.Font.FgRed ) ) - if (e.response.data.message != undefined) + if (message != undefined) { console.log( colorized.out( - `Inner exception message : "${e.response.data.message}"`, + `Inner exception message : "${message}"`, colorized.color.Font.FgRed ) ) - if ( - e.response.status == 400 || - e.response.status == 401 || - e.response.status == 402 || - e.response.status == 403 - ) + } + if ([400, 401, 402, 403].indexOf(status !== 'undefined')) { console.log( colorized.out( `Auth on endpoint is not implemented on this gatsby-source plugin.`, colorized.color.Font.FgRed ) ) + } } /** From 488cbd558a70a97d6b86fec6cd945ea4ebe2223b Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Sun, 6 Aug 2017 11:25:57 +0200 Subject: [PATCH 03/15] Add function that gets multiple WP pages --- .../src/gatsby-node.js | 113 ++++++++++++------ 1 file changed, 77 insertions(+), 36 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 4b8be3e61755e..45f9fb29ff2a7 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -1,5 +1,6 @@ const axios = require(`axios`) const crypto = require(`crypto`) +const querystring = require('querystring') const _ = require(`lodash`) const stringify = require(`json-stringify-safe`) const colorized = require(`./output-color`) @@ -100,7 +101,25 @@ exports.sourceNodes = async ( .forEach(n => touchNode(n.id)) // Call the main API Route to discover the all the routes exposed on this API. - let allRoutes = await axiosHelper(url) + let allRoutes + try { + let options = { + method: `get`, + url: url, + } + if (_auth != undefined) { + options.auth = { + username: _auth.user, + password: _auth.pass, + } + } + allRoutes = await axios({ + method: `get`, + url: url, + }) + } catch (e) { + httpExceptionHandler(e) + } if (allRoutes != undefined) { let validRoutes = getValidRoutes(allRoutes, url, baseUrl) @@ -141,31 +160,54 @@ exports.sourceNodes = async ( return } -/** - * Helper for axios GET with auth - * - * @param {any} url - * @param {any} auth - * @returns - */ -async function axiosHelper(url) { - let result +async function axiosPaginator (url, perPage = 10, page = 1) { try { - let options = { - method: `get`, - url: url, - } - if (_auth != undefined) { - options.auth = { - username: _auth.user, - password: _auth.pass, + let result = [] + + const getOptions = (perPage, page) => { + return { + method: `get`, + url: `${url}?${querystring.stringify({ 'per_page': perPage, 'page': page })}` } } - result = await axios(options) + + // Initial request gets the first page of data + // but also the total count of objects, used for + // multiple concurrent requests (rather than waterfall) + const options = getOptions(perPage, page) + const response = await axios(options) + + result = result.concat(response.data) + + // Get total number of entities + const total = parseInt(response.headers['x-wp-total']) + const totalPages = Math.ceil(total / perPage) + + if (_verbose) { + console.log(`\nTotal entities :`, total) + console.log(`Pages to be requested :`, totalPages) + } + + if (total < perPage) { + return result + } + + // For each X entities, make an HTTP request to page N + const requests = _.range(2, totalPages + 1).map((getPage) => { + const options = getOptions(perPage, getPage) + return axios(options) + }) + + return Promise.all(requests).then(pages => { + const data = pages.map(page => page.data) + data.forEach(postList => { + result = result.concat(postList) + }) + return result + }) } catch (e) { httpExceptionHandler(e) } - return result } /** @@ -201,13 +243,13 @@ function httpExceptionHandler (e) { /** * Extract valid routes and format its data. - * - * @param {any} allRoutes - * @param {any} url - * @param {any} baseUrl - * @returns + * + * @param {any} allRoutes + * @param {any} url + * @param {any} baseUrl + * @returns */ -function getValidRoutes(allRoutes, url, baseUrl) { +function getValidRoutes (allRoutes, url, baseUrl) { let validRoutes = [] for (let key of Object.keys(allRoutes.data.routes)) { @@ -328,7 +370,7 @@ const getManufacturer = route => * * @param {any} route * @param {any} createNode - * @param {any} parentNodeId (Optionnal parent node ID) + * @param {any} parentNodeId (Optional parent node ID) */ async function fetchData(route, createNode, parentNodeId) { const type = route.type @@ -350,17 +392,17 @@ async function fetchData(route, createNode, parentNodeId) { if (_verbose) console.time(`Fetching the ${type} took`) } - let routeResponse = await axiosHelper(url) + const routeResponse = await axiosPaginator(url, 40, 1) if (routeResponse) { // Process entities to creating GraphQL Nodes. - if (Array.isArray(routeResponse.data)) { - for (let ent of routeResponse.data) { + if (Array.isArray(routeResponse)) { + for (let ent of routeResponse) { await createGraphQLNode(ent, type, createNode, parentNodeId) } } else { await createGraphQLNode( - routeResponse.data, + routeResponse, type, createNode, parentNodeId @@ -371,15 +413,14 @@ async function fetchData(route, createNode, parentNodeId) { let length if ( routeResponse != undefined && - routeResponse.data != undefined && - Array.isArray(routeResponse.data) + Array.isArray(routeResponse) ) { - length = routeResponse.data.length + length = routeResponse.length } else if ( - routeResponse.data != undefined && + routeResponse != undefined && !Array.isArray(routeResponse.data) ) { - length = Object.keys(routeResponse.data).length + length = Object.keys(routeResponse).length } console.log( colorized.out(`${type} fetched : ${length}`, colorized.color.Font.FgGreen) From f48802056eb0a72c9fc137b4ed71025f273c3846 Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Sun, 6 Aug 2017 11:36:06 +0200 Subject: [PATCH 04/15] Allow perPage to be configurable (with default of 10) --- .../src/gatsby-node.js | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 45f9fb29ff2a7..44f9460448c44 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -18,6 +18,7 @@ let _getNode let _useACF let _hostingWPCOM let _auth +let _perPage let _parentChildNodes = [] @@ -31,7 +32,7 @@ const refactoredEntityTypes = { // ========= Main =========== exports.sourceNodes = async ( { boundActionCreators, getNode, hasNodeChanged, store }, - { baseUrl, protocol, hostingWPCOM, useACF, auth, verboseOutput } + { baseUrl, protocol, hostingWPCOM, useACF, auth, verboseOutput, perPage = 10 } ) => { const { createNode, @@ -45,6 +46,7 @@ exports.sourceNodes = async ( _useACF = useACF _hostingWPCOM = hostingWPCOM _auth = auth + _perPage = perPage // If the site is hosted on wordpress.com, the API Route differs. // Same entity types are exposed (excepted for medias and users which need auth) @@ -348,8 +350,8 @@ function getValidRoutes (allRoutes, url, baseUrl) { /** * Extract the raw entity type from route - * - * @param {any} route + * + * @param {any} route */ const getRawEntityType = route => route._links.self.substring( @@ -359,8 +361,8 @@ const getRawEntityType = route => /** * Extract the route manufacturer - * - * @param {any} route + * + * @param {any} route */ const getManufacturer = route => route.namespace.substring(0, route.namespace.lastIndexOf(`/`)) @@ -392,7 +394,7 @@ async function fetchData(route, createNode, parentNodeId) { if (_verbose) console.time(`Fetching the ${type} took`) } - const routeResponse = await axiosPaginator(url, 40, 1) + const routeResponse = await axiosPaginator(url, _perPage, 1) if (routeResponse) { // Process entities to creating GraphQL Nodes. @@ -433,8 +435,8 @@ async function fetchData(route, createNode, parentNodeId) { /** * Encrypts a String using md5 hash of hexadecimal digest. - * - * @param {any} str + * + * @param {any} str */ const digest = str => crypto.createHash(`md5`).update(str).digest(`hex`) @@ -494,9 +496,9 @@ function createGraphQLNode(ent, type, createNode, parentNodeId) { /** * Loop through fields to validate naming conventions and extract child nodes. - * + * * @param {any} ent - * @param {any} newEnt + * @param {any} newEnt * @returns the new entity with fields */ function addFields(ent, newEnt, createNode) { @@ -535,9 +537,9 @@ function addFields(ent, newEnt, createNode) { /** * Add fields recursively - * - * @param {any} ent - * @param {any} newEnt + * + * @param {any} ent + * @param {any} newEnt * @returns the new node */ function recursiveAddFields(ent, newEnt) { @@ -566,8 +568,8 @@ function recursiveAddFields(ent, newEnt) { /** * Validate the GraphQL naming convetions & protect specific fields. - * - * @param {any} key + * + * @param {any} key * @returns the valid name */ function getValidName(key) { From b0c8367a0a4a38528d2facdb6937c4744c6c195f Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Sun, 6 Aug 2017 11:43:23 +0200 Subject: [PATCH 05/15] Add auth to paginated requests --- packages/gatsby-source-wordpress/src/gatsby-node.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 44f9460448c44..be77924c58440 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -169,7 +169,10 @@ async function axiosPaginator (url, perPage = 10, page = 1) { const getOptions = (perPage, page) => { return { method: `get`, - url: `${url}?${querystring.stringify({ 'per_page': perPage, 'page': page })}` + url: `${url}?${querystring.stringify({ 'per_page': perPage, 'page': page })}`, + auth: _auth + ? { username: _auth.user, password: _auth.pass } + : null } } From b6c056af085e14d0b255af8e1cd041444dcbba32 Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Sun, 6 Aug 2017 11:46:47 +0200 Subject: [PATCH 06/15] Rename pagination function --- packages/gatsby-source-wordpress/src/gatsby-node.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index be77924c58440..5156522d4c5cd 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -162,7 +162,7 @@ exports.sourceNodes = async ( return } -async function axiosPaginator (url, perPage = 10, page = 1) { +async function getPages (url, perPage = 10, page = 1) { try { let result = [] @@ -397,7 +397,7 @@ async function fetchData(route, createNode, parentNodeId) { if (_verbose) console.time(`Fetching the ${type} took`) } - const routeResponse = await axiosPaginator(url, _perPage, 1) + const routeResponse = await getPages(url, _perPage, 1) if (routeResponse) { // Process entities to creating GraphQL Nodes. From b215a10e75fca8a16b55bd34c741c01b751667e6 Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Sun, 6 Aug 2017 12:28:51 +0200 Subject: [PATCH 07/15] Fix exception handler --- packages/gatsby-source-wordpress/src/gatsby-node.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 5156522d4c5cd..027a51f386113 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -228,7 +228,7 @@ function httpExceptionHandler (e) { colorized.color.Font.FgRed ) ) - if (message != undefined) { + if (message) { console.log( colorized.out( `Inner exception message : "${message}"`, @@ -236,7 +236,7 @@ function httpExceptionHandler (e) { ) ) } - if ([400, 401, 402, 403].indexOf(status !== 'undefined')) { + if ([400, 401, 402, 403].includes(status)) { console.log( colorized.out( `Auth on endpoint is not implemented on this gatsby-source plugin.`, From d969fa8e7adc3b94486e0cbbb0807fd13cc57ac6 Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Sun, 6 Aug 2017 21:02:50 +0200 Subject: [PATCH 08/15] Lint gatsby-node.js --- .../src/gatsby-node.js | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 027a51f386113..be766dfbb01f8 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -1,6 +1,6 @@ const axios = require(`axios`) const crypto = require(`crypto`) -const querystring = require('querystring') +const querystring = require(`querystring`) const _ = require(`lodash`) const stringify = require(`json-stringify-safe`) const colorized = require(`./output-color`) @@ -162,17 +162,18 @@ exports.sourceNodes = async ( return } -async function getPages (url, perPage = 10, page = 1) { +async function getPages(url, perPage = 10, page = 1) { try { let result = [] const getOptions = (perPage, page) => { return { method: `get`, - url: `${url}?${querystring.stringify({ 'per_page': perPage, 'page': page })}`, - auth: _auth - ? { username: _auth.user, password: _auth.pass } - : null + url: `${url}?${querystring.stringify({ + per_page: perPage, + page: page, + })}`, + auth: _auth ? { username: _auth.user, password: _auth.pass } : null, } } @@ -185,7 +186,7 @@ async function getPages (url, perPage = 10, page = 1) { result = result.concat(response.data) // Get total number of entities - const total = parseInt(response.headers['x-wp-total']) + const total = parseInt(response.headers[`x-wp-total`]) const totalPages = Math.ceil(total / perPage) if (_verbose) { @@ -198,7 +199,7 @@ async function getPages (url, perPage = 10, page = 1) { } // For each X entities, make an HTTP request to page N - const requests = _.range(2, totalPages + 1).map((getPage) => { + const requests = _.range(2, totalPages + 1).map(getPage => { const options = getOptions(perPage, getPage) return axios(options) }) @@ -220,7 +221,7 @@ async function getPages (url, perPage = 10, page = 1) { * * @param {any} e */ -function httpExceptionHandler (e) { +function httpExceptionHandler(e) { const { status, statusText, data: { message } } = e.response console.log( colorized.out( @@ -254,7 +255,7 @@ function httpExceptionHandler (e) { * @param {any} baseUrl * @returns */ -function getValidRoutes (allRoutes, url, baseUrl) { +function getValidRoutes(allRoutes, url, baseUrl) { let validRoutes = [] for (let key of Object.keys(allRoutes.data.routes)) { @@ -406,20 +407,12 @@ async function fetchData(route, createNode, parentNodeId) { await createGraphQLNode(ent, type, createNode, parentNodeId) } } else { - await createGraphQLNode( - routeResponse, - type, - createNode, - parentNodeId - ) + await createGraphQLNode(routeResponse, type, createNode, parentNodeId) } // TODO : Get the number of created nodes using the nodes in state. let length - if ( - routeResponse != undefined && - Array.isArray(routeResponse) - ) { + if (routeResponse != undefined && Array.isArray(routeResponse)) { length = routeResponse.length } else if ( routeResponse != undefined && From 973d85b08977893092694187d770e0fbf81aa82c Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Wed, 9 Aug 2017 10:23:28 +0200 Subject: [PATCH 09/15] Add babel dependencies/config --- packages/gatsby-source-wordpress/package.json | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-wordpress/package.json b/packages/gatsby-source-wordpress/package.json index 86ba8a7bd3266..c798566047ec8 100644 --- a/packages/gatsby-source-wordpress/package.json +++ b/packages/gatsby-source-wordpress/package.json @@ -8,6 +8,13 @@ "lodash": "^4.17.4", "qs": "^6.4.0" }, + "devDependencies": { + "babel-cli": "^6.24.1", + "babel-plugin-transform-runtime": "^6.23.0", + "babel-preset-es2015": "^6.24.1", + "babel-preset-stage-0": "^6.24.1", + "babel-runtime": "^6.25.0" + }, "deprecated": false, "description": "Gatsby source plugin for building websites using the Wordpress CMS as a data source.", "keywords": [ @@ -21,5 +28,20 @@ "build": "babel src --out-dir .", "watch": "babel -w src --out-dir ." }, - "version": "1.6.3" + "version": "1.6.3", + "babel": { + "presets": [ + "es2015", + "stage-0" + ], + "plugins": [ + [ + "transform-runtime", + { + "polyfill": false, + "regenerator": true + } + ] + ] + } } From d01c9dfa647fa79dc60db65b4c1bf50a8b96aa4d Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Wed, 9 Aug 2017 10:27:19 +0200 Subject: [PATCH 10/15] =?UTF-8?q?Don=E2=80=99t=20pass=20perPage=20paramete?= =?UTF-8?q?r=20around?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gatsby-source-wordpress/src/gatsby-node.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index be766dfbb01f8..0cbf0b88e0fac 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -162,15 +162,15 @@ exports.sourceNodes = async ( return } -async function getPages(url, perPage = 10, page = 1) { +async function getPages(url, page = 1) { try { let result = [] - const getOptions = (perPage, page) => { + const getOptions = (page) => { return { method: `get`, url: `${url}?${querystring.stringify({ - per_page: perPage, + per_page: _perPage, page: page, })}`, auth: _auth ? { username: _auth.user, password: _auth.pass } : null, @@ -180,27 +180,27 @@ async function getPages(url, perPage = 10, page = 1) { // Initial request gets the first page of data // but also the total count of objects, used for // multiple concurrent requests (rather than waterfall) - const options = getOptions(perPage, page) + const options = getOptions(page) const response = await axios(options) result = result.concat(response.data) // Get total number of entities const total = parseInt(response.headers[`x-wp-total`]) - const totalPages = Math.ceil(total / perPage) + const totalPages = Math.ceil(total / _perPage) if (_verbose) { console.log(`\nTotal entities :`, total) console.log(`Pages to be requested :`, totalPages) } - if (total < perPage) { + if (total < _perPage) { return result } // For each X entities, make an HTTP request to page N const requests = _.range(2, totalPages + 1).map(getPage => { - const options = getOptions(perPage, getPage) + const options = getOptions(getPage) return axios(options) }) @@ -398,7 +398,7 @@ async function fetchData(route, createNode, parentNodeId) { if (_verbose) console.time(`Fetching the ${type} took`) } - const routeResponse = await getPages(url, _perPage, 1) + const routeResponse = await getPages(url, 1) if (routeResponse) { // Process entities to creating GraphQL Nodes. From 188e83e6af1651123fc752a88bc53b9f0b629d62 Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Wed, 9 Aug 2017 10:27:41 +0200 Subject: [PATCH 11/15] Remove unused variable --- packages/gatsby-source-wordpress/src/gatsby-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 0cbf0b88e0fac..4d8ea0194d444 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -31,7 +31,7 @@ const refactoredEntityTypes = { // ========= Main =========== exports.sourceNodes = async ( - { boundActionCreators, getNode, hasNodeChanged, store }, + { boundActionCreators, getNode, store }, { baseUrl, protocol, hostingWPCOM, useACF, auth, verboseOutput, perPage = 10 } ) => { const { From 7c2bd734eba21e106bbedb3a36384c7d64452d7c Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Wed, 9 Aug 2017 10:32:38 +0200 Subject: [PATCH 12/15] Code cleanup --- .../src/gatsby-node.js | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 4d8ea0194d444..4f0b7eb27393d 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -212,7 +212,7 @@ async function getPages(url, page = 1) { return result }) } catch (e) { - httpExceptionHandler(e) + return httpExceptionHandler(e) } } @@ -290,7 +290,7 @@ function getValidRoutes(allRoutes, url, baseUrl) { const manufacturer = getManufacturer(route) let rawType = `` - if (manufacturer == `wp`) { + if (manufacturer === `wp`) { rawType = `${typePrefix}${entityType}` } @@ -412,12 +412,9 @@ async function fetchData(route, createNode, parentNodeId) { // TODO : Get the number of created nodes using the nodes in state. let length - if (routeResponse != undefined && Array.isArray(routeResponse)) { + if (routeResponse && Array.isArray(routeResponse)) { length = routeResponse.length - } else if ( - routeResponse != undefined && - !Array.isArray(routeResponse.data) - ) { + } else if (routeResponse && !Array.isArray(routeResponse)) { length = Object.keys(routeResponse).length } console.log( @@ -425,8 +422,9 @@ async function fetchData(route, createNode, parentNodeId) { ) } - if (_verbose && parentNodeId == undefined) + if (_verbose && !parentNodeId) { console.timeEnd(`Fetching the ${type} took`) + } } /** @@ -445,7 +443,7 @@ const digest = str => crypto.createHash(`md5`).update(str).digest(`hex`) * @param {any} parentNodeId (Optionnal parent node ID) */ function createGraphQLNode(ent, type, createNode, parentNodeId) { - let id = ent.id == undefined ? (ent.ID == undefined ? 0 : ent.ID) : ent.id + let id = !ent.id ? (!ent.ID ? 0 : ent.ID) : ent.id let node = { id: `${type}_${id.toString()}`, children: [], @@ -457,35 +455,33 @@ function createGraphQLNode(ent, type, createNode, parentNodeId) { }, } - if (type == refactoredEntityTypes.post) { + if (type === refactoredEntityTypes.post) { node.id = `POST_${ent.id.toString()}` node.internal.type = refactoredEntityTypes.post - } else if (type == refactoredEntityTypes.page) { + } else if (type === refactoredEntityTypes.page) { node.id = `PAGE_${ent.id.toString()}` node.internal.type = refactoredEntityTypes.page - } else if (type == refactoredEntityTypes.tag) { + } else if (type === refactoredEntityTypes.tag) { node.id = `TAG_${ent.id.toString()}` node.internal.type = refactoredEntityTypes.tag - } else if (type == refactoredEntityTypes.category) { + } else if (type === refactoredEntityTypes.category) { node.id = `CATEGORY_${ent.id.toString()}` node.internal.type = refactoredEntityTypes.category } node = addFields(ent, node, createNode) - if ( - type == refactoredEntityTypes.post || - type == refactoredEntityTypes.page - ) { + if (type === refactoredEntityTypes.post || Ztype === refactoredEntityTypes.page) { // TODO : Move this to field recursive and add other fields that have rendered field node.title = ent.title.rendered node.content = ent.content.rendered node.excerpt = ent.excerpt.rendered } + node.internal.contentDigest = digest(stringify(node)) createNode(node) - if (parentNodeId != undefined) { + if (parentNodeId) { _parentChildNodes.push({ parentId: parentNodeId, childNodeId: node.id }) } } @@ -495,6 +491,7 @@ function createGraphQLNode(ent, type, createNode, parentNodeId) { * * @param {any} ent * @param {any} newEnt + * @param {function} createNode * @returns the new entity with fields */ function addFields(ent, newEnt, createNode) { @@ -516,11 +513,7 @@ function addFields(ent, newEnt, createNode) { acfNode.internal.contentDigest = digest(stringify(acfNode)) createNode(acfNode) _parentChildNodes.push({ parentId: newEnt.id, childNodeId: acfNode.id }) - } else if ( - newEnt.meta != undefined && - newEnt.meta.links != undefined && - newEnt.meta.links.self != undefined - ) { + } else if (newEnt.meta && newEnt.meta.links && newEnt.meta.links.self) { //The entity as a link to more content for this entity fetchData( { url: newEnt.meta.links.self, type: `${newEnt.internal.type}_Extended` }, @@ -545,11 +538,11 @@ function recursiveAddFields(ent, newEnt) { if (key !== `acf`) { newEnt[key] = ent[k] // Nested Objects & Arrays of Objects - if (typeof ent[key] == `object`) { + if (typeof ent[key] === `object`) { if (!Array.isArray(ent[key]) && ent[key] != null) { newEnt[key] = recursiveAddFields(ent[key], {}) } else if (Array.isArray(ent[key])) { - if (ent[key].length > 0 && typeof ent[key][0] == `object`) { + if (ent[key].length > 0 && typeof ent[key][0] === `object`) { ent[k].map((el, i) => { newEnt[key][i] = recursiveAddFields(el, {}) }) From 56ca578d028d96d1bcda391d01fc690e9798f6b1 Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Wed, 9 Aug 2017 10:33:07 +0200 Subject: [PATCH 13/15] Default `per_page` to 100 --- packages/gatsby-source-wordpress/src/gatsby-node.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 4f0b7eb27393d..26044ca2ff251 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -32,7 +32,7 @@ const refactoredEntityTypes = { // ========= Main =========== exports.sourceNodes = async ( { boundActionCreators, getNode, store }, - { baseUrl, protocol, hostingWPCOM, useACF, auth, verboseOutput, perPage = 10 } + { baseUrl, protocol, hostingWPCOM, useACF, auth, verboseOutput, perPage = 100 } ) => { const { createNode, @@ -471,7 +471,7 @@ function createGraphQLNode(ent, type, createNode, parentNodeId) { node = addFields(ent, node, createNode) - if (type === refactoredEntityTypes.post || Ztype === refactoredEntityTypes.page) { + if (type === refactoredEntityTypes.post || type === refactoredEntityTypes.page) { // TODO : Move this to field recursive and add other fields that have rendered field node.title = ent.title.rendered node.content = ent.content.rendered From 564143d73389138f0a66ae1057198c11943d934f Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Wed, 9 Aug 2017 10:54:56 +0200 Subject: [PATCH 14/15] Use WP total pages header & code style and style format --- .../src/gatsby-node.js | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/gatsby-node.js b/packages/gatsby-source-wordpress/src/gatsby-node.js index 26044ca2ff251..f53d1b768d7bb 100644 --- a/packages/gatsby-source-wordpress/src/gatsby-node.js +++ b/packages/gatsby-source-wordpress/src/gatsby-node.js @@ -32,7 +32,15 @@ const refactoredEntityTypes = { // ========= Main =========== exports.sourceNodes = async ( { boundActionCreators, getNode, store }, - { baseUrl, protocol, hostingWPCOM, useACF, auth, verboseOutput, perPage = 100 } + { + baseUrl, + protocol, + hostingWPCOM, + useACF, + auth, + verboseOutput, + perPage = 100, + } ) => { const { createNode, @@ -109,7 +117,7 @@ exports.sourceNodes = async ( method: `get`, url: url, } - if (_auth != undefined) { + if (_auth) { options.auth = { username: _auth.user, password: _auth.pass, @@ -123,7 +131,7 @@ exports.sourceNodes = async ( httpExceptionHandler(e) } - if (allRoutes != undefined) { + if (allRoutes) { let validRoutes = getValidRoutes(allRoutes, url, baseUrl) console.log(``) @@ -166,7 +174,7 @@ async function getPages(url, page = 1) { try { let result = [] - const getOptions = (page) => { + const getOptions = page => { return { method: `get`, url: `${url}?${querystring.stringify({ @@ -181,24 +189,26 @@ async function getPages(url, page = 1) { // but also the total count of objects, used for // multiple concurrent requests (rather than waterfall) const options = getOptions(page) - const response = await axios(options) + const { headers, data } = await axios(options) + + result = result.concat(data) - result = result.concat(response.data) + // Some resources have no paging, e.g. `/types` + const wpTotal = headers[`x-wp-total`] - // Get total number of entities - const total = parseInt(response.headers[`x-wp-total`]) - const totalPages = Math.ceil(total / _perPage) + const total = parseInt(wpTotal) + const totalPages = parseInt(headers[`x-wp-totalpages`]) + + if (!wpTotal || totalPages <= 1) { + return result + } if (_verbose) { console.log(`\nTotal entities :`, total) console.log(`Pages to be requested :`, totalPages) } - if (total < _perPage) { - return result - } - - // For each X entities, make an HTTP request to page N + // We got page 1, now we want pages 2 through totalPages const requests = _.range(2, totalPages + 1).map(getPage => { const options = getOptions(getPage) return axios(options) @@ -206,8 +216,8 @@ async function getPages(url, page = 1) { return Promise.all(requests).then(pages => { const data = pages.map(page => page.data) - data.forEach(postList => { - result = result.concat(postList) + data.forEach(list => { + result = result.concat(list) }) return result }) @@ -471,7 +481,10 @@ function createGraphQLNode(ent, type, createNode, parentNodeId) { node = addFields(ent, node, createNode) - if (type === refactoredEntityTypes.post || type === refactoredEntityTypes.page) { + if ( + type === refactoredEntityTypes.post || + type === refactoredEntityTypes.page + ) { // TODO : Move this to field recursive and add other fields that have rendered field node.title = ent.title.rendered node.content = ent.content.rendered From f10625f398f6784321dcc56ca6328e2bea0be53e Mon Sep 17 00:00:00 2001 From: Alex Bass Date: Thu, 10 Aug 2017 15:46:55 +0200 Subject: [PATCH 15/15] Revert "Add babel dependencies/config" This reverts commit 973d85b08977893092694187d770e0fbf81aa82c. --- packages/gatsby-source-wordpress/package.json | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/packages/gatsby-source-wordpress/package.json b/packages/gatsby-source-wordpress/package.json index c798566047ec8..86ba8a7bd3266 100644 --- a/packages/gatsby-source-wordpress/package.json +++ b/packages/gatsby-source-wordpress/package.json @@ -8,13 +8,6 @@ "lodash": "^4.17.4", "qs": "^6.4.0" }, - "devDependencies": { - "babel-cli": "^6.24.1", - "babel-plugin-transform-runtime": "^6.23.0", - "babel-preset-es2015": "^6.24.1", - "babel-preset-stage-0": "^6.24.1", - "babel-runtime": "^6.25.0" - }, "deprecated": false, "description": "Gatsby source plugin for building websites using the Wordpress CMS as a data source.", "keywords": [ @@ -28,20 +21,5 @@ "build": "babel src --out-dir .", "watch": "babel -w src --out-dir ." }, - "version": "1.6.3", - "babel": { - "presets": [ - "es2015", - "stage-0" - ], - "plugins": [ - [ - "transform-runtime", - { - "polyfill": false, - "regenerator": true - } - ] - ] - } + "version": "1.6.3" }