From 8f6175db95ce7789f8694fe43f13a4f445615a17 Mon Sep 17 00:00:00 2001 From: Thomas Smith Date: Tue, 5 Jun 2018 20:37:52 +0100 Subject: [PATCH 1/3] Use Promises to prevent calling done too early --- .../gatsby-source-mongodb/src/gatsby-node.js | 135 ++++++++++-------- yarn.lock | 13 ++ 2 files changed, 85 insertions(+), 63 deletions(-) diff --git a/packages/gatsby-source-mongodb/src/gatsby-node.js b/packages/gatsby-source-mongodb/src/gatsby-node.js index 0cfe7cd21be80..5cbc4b43bde4d 100644 --- a/packages/gatsby-source-mongodb/src/gatsby-node.js +++ b/packages/gatsby-source-mongodb/src/gatsby-node.js @@ -1,3 +1,4 @@ +/* eslint-disable */ const MongoClient = require(`mongodb`).MongoClient const crypto = require(`crypto`) const prepareMappingChildNode = require(`./mapping`) @@ -30,13 +31,19 @@ exports.sourceNodes = ( return } let collection = pluginOptions.collection || `documents` - if (_.isArray(collection)) { - for (const col of collection) { - createNodes(db, pluginOptions, dbName, createNode, col, done) - } - } else { - createNodes(db, pluginOptions, dbName, createNode, collection, done) + if (!_.isArray(collection)) { + collection = [collection] + } + + const promises = [] + for (const col of collection) { + const promise = createNodes(db, pluginOptions, dbName, createNode, col) + promises.push(promise) } + + Promise.all(promises) + .then(() => done()) + .catch((e) => console.warn(e)) } ) } @@ -46,71 +53,73 @@ function createNodes( pluginOptions, dbName, createNode, - collectionName, - done + collectionName ) { - let collection = db.collection(collectionName) - let cursor = collection.find() - // Execute the each command, triggers for each document - cursor.each(function(err, item) { - // If the item is null then the cursor is exhausted/empty and closed - if (item == null) { - // Let's close the db - db.close() - done() - } else { - var id = item._id.toString() - delete item._id + return new Promise((resolve, reject) => { + let collection = db.collection(collectionName) + let cursor = collection.find() - var node = { - // Data for the node. - ...item, - id: `${id}`, - parent: `__${collectionName}__`, - children: [], - internal: { - type: `mongodb${caps(dbName)}${caps(collectionName)}`, - content: JSON.stringify(item), - contentDigest: crypto - .createHash(`md5`) - .update(JSON.stringify(item)) - .digest(`hex`), - }, - } - const childrenNodes = [] - if (pluginOptions.map) { - let mapObj = pluginOptions.map - if (pluginOptions.map[collectionName]) { - mapObj = pluginOptions.map[collectionName] + // Execute the each command, triggers for each document + cursor.each(function(err, item) { + // If the item is null then the cursor is exhausted/empty and closed + if (item == null) { + // Let's close the db + db.close() + resolve() + } else { + var id = item._id.toString() + delete item._id + + var node = { + // Data for the node. + ...item, + id: `${id}`, + parent: `__${collectionName}__`, + children: [], + internal: { + type: `mongodb${caps(dbName)}${caps(collectionName)}`, + content: JSON.stringify(item), + contentDigest: crypto + .createHash(`md5`) + .update(JSON.stringify(item)) + .digest(`hex`), + }, } - // We need to map certain fields to a contenttype. - Object.keys(mapObj).forEach(mediaItemFieldKey => { - if ( - node[mediaItemFieldKey] && - (typeof mapObj[mediaItemFieldKey] === `string` || - mapObj[mediaItemFieldKey] instanceof String) - ) { - const mappingChildNode = prepareMappingChildNode( - node, - mediaItemFieldKey, - node[mediaItemFieldKey], - mapObj[mediaItemFieldKey], - createNode - ) + const childrenNodes = [] + if (pluginOptions.map) { + let mapObj = pluginOptions.map + if (pluginOptions.map[collectionName]) { + mapObj = pluginOptions.map[collectionName] + } + // We need to map certain fields to a contenttype. + Object.keys(mapObj).forEach(mediaItemFieldKey => { + if ( + node[mediaItemFieldKey] && + (typeof mapObj[mediaItemFieldKey] === `string` || + mapObj[mediaItemFieldKey] instanceof String) + ) { + const mappingChildNode = prepareMappingChildNode( + node, + mediaItemFieldKey, + node[mediaItemFieldKey], + mapObj[mediaItemFieldKey], + createNode + ) - node[`${mediaItemFieldKey}___NODE`] = mappingChildNode.id - childrenNodes.push(mappingChildNode) + node[`${mediaItemFieldKey}___NODE`] = mappingChildNode.id + childrenNodes.push(mappingChildNode) - delete node[mediaItemFieldKey] - } + delete node[mediaItemFieldKey] + } + }) + } + createNode(node) + childrenNodes.forEach(node => { + createNode(node) }) } - createNode(node) - childrenNodes.forEach(node => { - createNode(node) - }) - } + }) }) } diff --git a/yarn.lock b/yarn.lock index bc069dd07b98b..44e1710ec2107 100644 --- a/yarn.lock +++ b/yarn.lock @@ -658,6 +658,13 @@ axios@^0.17.1: follow-redirects "^1.2.5" is-buffer "^1.1.5" +axios@^0.18.0: + version "0.18.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102" + dependencies: + follow-redirects "^1.3.0" + is-buffer "^1.1.5" + axobject-query@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-0.1.0.tgz#62f59dbc59c9f9242759ca349960e7a2fe3c36c0" @@ -5081,6 +5088,12 @@ follow-redirects@^1.2.3, follow-redirects@^1.2.5: dependencies: debug "^3.1.0" +follow-redirects@^1.3.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.0.tgz#234f49cf770b7f35b40e790f636ceba0c3a0ab77" + dependencies: + debug "^3.1.0" + for-each@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.2.tgz#2c40450b9348e97f281322593ba96704b9abd4d4" From 8eccde90c68deade79b49e7dea6cdfeaacebec1b Mon Sep 17 00:00:00 2001 From: Thomas Smith Date: Tue, 5 Jun 2018 20:44:55 +0100 Subject: [PATCH 2/3] Remove comment --- packages/gatsby-source-mongodb/src/gatsby-node.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gatsby-source-mongodb/src/gatsby-node.js b/packages/gatsby-source-mongodb/src/gatsby-node.js index 5cbc4b43bde4d..3823ff1db69b0 100644 --- a/packages/gatsby-source-mongodb/src/gatsby-node.js +++ b/packages/gatsby-source-mongodb/src/gatsby-node.js @@ -1,4 +1,3 @@ -/* eslint-disable */ const MongoClient = require(`mongodb`).MongoClient const crypto = require(`crypto`) const prepareMappingChildNode = require(`./mapping`) From 677f54421ea83539f4ea4b7711ea3520d6777b93 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 5 Jun 2018 23:29:06 +0200 Subject: [PATCH 3/3] promises all the way --- .../gatsby-source-mongodb/src/gatsby-node.js | 51 +++++++------------ 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/packages/gatsby-source-mongodb/src/gatsby-node.js b/packages/gatsby-source-mongodb/src/gatsby-node.js index 3823ff1db69b0..72f9b8ad246f8 100644 --- a/packages/gatsby-source-mongodb/src/gatsby-node.js +++ b/packages/gatsby-source-mongodb/src/gatsby-node.js @@ -5,8 +5,7 @@ const _ = require(`lodash`) exports.sourceNodes = ( { boundActionCreators, getNode, hasNodeChanged }, - pluginOptions, - done + pluginOptions ) => { const { createNode } = boundActionCreators @@ -19,42 +18,30 @@ exports.sourceNodes = ( if (pluginOptions.auth) authUrlPart = `${pluginOptions.auth.user}:${pluginOptions.auth.password}@` - MongoClient.connect( - `mongodb://${authUrlPart}${serverOptions.address}:${ - serverOptions.port - }/${dbName}`, - function(err, db) { - // Establish connection to db - if (err) { - console.warn(err) - return - } - let collection = pluginOptions.collection || `documents` + const connectionURL = `mongodb://${authUrlPart}${serverOptions.address}:${ + serverOptions.port + }/${dbName}` + + return MongoClient.connect(connectionURL) + .then(db => { + let collection = pluginOptions.collection || [`documents`] if (!_.isArray(collection)) { collection = [collection] } - const promises = [] - for (const col of collection) { - const promise = createNodes(db, pluginOptions, dbName, createNode, col) - promises.push(promise) - } - - Promise.all(promises) - .then(() => done()) - .catch((e) => console.warn(e)) - } - ) + return Promise.all( + collection.map(col => + createNodes(db, pluginOptions, dbName, createNode, col) + ) + ) + }) + .catch(err => { + console.warn(err) + return err + }) } -function createNodes( - db, - pluginOptions, - dbName, - createNode, - collectionName -) { - +function createNodes(db, pluginOptions, dbName, createNode, collectionName) { return new Promise((resolve, reject) => { let collection = db.collection(collectionName) let cursor = collection.find()