From 8b96f950b22d3688d8e2a800109fd4ae902917e9 Mon Sep 17 00:00:00 2001 From: Peter van der Zee <209817+pvdz@users.noreply.github.com> Date: Wed, 1 Apr 2020 15:10:42 +0200 Subject: [PATCH] chore(gatsby): create single filter cache key generator (#22716) * chore(gatsby): create single filter cache key generator * Add comment about future proofing --- packages/gatsby/src/redux/nodes.ts | 26 +++++--------- packages/gatsby/src/redux/run-sift.js | 52 ++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/packages/gatsby/src/redux/nodes.ts b/packages/gatsby/src/redux/nodes.ts index 3385e108a6a8d..1c68274b3f17f 100644 --- a/packages/gatsby/src/redux/nodes.ts +++ b/packages/gatsby/src/redux/nodes.ts @@ -2,6 +2,8 @@ import { store } from "./" import { IGatsbyNode } from "./types" import { createPageDependency } from "./actions/add-page-dependency" +export type FilterCacheKey = string + /** * Get all nodes from redux store. */ @@ -155,27 +157,22 @@ export const addResolvedNodes = ( * looping over all the nodes, when the number of pages (/nodes) scale up. */ export const ensureIndexByTypedChain = ( + cacheKey: FilterCacheKey, chain: string[], nodeTypeNames: string[], typedKeyValueIndexes: Map< - string, + FilterCacheKey, Map> > ): void => { - const chained = chain.join(`+`) - - const nodeTypeNamePrefix = nodeTypeNames.join(`,`) + `/` - // The format of the typedKey is `type,type/path+to+eqobj` - const typedKey = nodeTypeNamePrefix + chained - - if (typedKeyValueIndexes.has(typedKey)) { + if (typedKeyValueIndexes.has(cacheKey)) { return } const { nodes, resolvedNodesCache } = store.getState() const byKeyValue = new Map>() - typedKeyValueIndexes.set(typedKey, byKeyValue) + typedKeyValueIndexes.set(cacheKey, byKeyValue) nodes.forEach(node => { if (!nodeTypeNames.includes(node.internal.type)) { @@ -232,18 +229,13 @@ export const ensureIndexByTypedChain = ( * per `id` so there's a minor optimization for that (no need for Sets). */ export const getNodesByTypedChain = ( - chain: string[], + cacheKey: FilterCacheKey, value: boolean | number | string, - nodeTypeNames: string[], typedKeyValueIndexes: Map< - string, + FilterCacheKey, Map> > ): Set | undefined => { - const key = chain.join(`+`) - - const typedKey = nodeTypeNames.join(`,`) + `/` + key - - const byTypedKey = typedKeyValueIndexes?.get(typedKey) + const byTypedKey = typedKeyValueIndexes?.get(cacheKey) return byTypedKey?.get(value) } diff --git a/packages/gatsby/src/redux/run-sift.js b/packages/gatsby/src/redux/run-sift.js index e1905a2c21010..aba8ae2936406 100644 --- a/packages/gatsby/src/redux/run-sift.js +++ b/packages/gatsby/src/redux/run-sift.js @@ -1,4 +1,5 @@ // @flow + const { default: sift } = require(`sift`) const { prepareRegex } = require(`../utils/prepare-regex`) const { makeRe } = require(`micromatch`) @@ -19,6 +20,35 @@ const { getNode: siftGetNode, } = require(`./nodes`) +/** + * Creates a key for the filterCache + * + * @param {Array} typeNames + * @param {DbQuery} filter + * @returns {FilterCacheKey} (a string: `types.join()/path.join()/operator` ) + */ +const createTypedFilterCacheKey = (typeNames, filter) => { + // Note: while `elemMatch` is a special case, in the key it's just `elemMatch` + // (This function is future proof for elemMatch support, won't receive it yet) + let f = filter + let comparator = `` + let paths /*: Array*/ = [] + while (f) { + paths.push(...f.path) + if (f.type === `elemMatch`) { + let q /*: IDbQueryElemMatch*/ = f + f = q.nestedQuery + } else { + let q /*: IDbQueryQuery*/ = f + comparator = q.query.comparator + break + } + } + + // Note: the separators (`,` and `/`) are arbitrary but must be different + return typeNames.join(`,`) + `/` + comparator + `/` + paths.join(`,`) +} + ///////////////////////////////////////////////////////////////////// // Parse filter ///////////////////////////////////////////////////////////////////// @@ -105,7 +135,7 @@ function handleMany(siftArgs, nodes) { * * @param {Array} filters Resolved. (Should be checked by caller to exist) * @param {Array} nodeTypeNames - * @param {Map>>} typedKeyValueIndexes + * @param {Map>>} typedKeyValueIndexes * @returns {Array | undefined} */ const runFlatFiltersWithoutSift = ( @@ -149,7 +179,7 @@ const runFlatFiltersWithoutSift = ( /** * @param {Array} filters * @param {Array} nodeTypeNames - * @param {Map>>} typedKeyValueIndexes + * @param {Map>>} typedKeyValueIndexes * @returns {Array> | undefined} Undefined means at least one * cache was not found. Must fallback to sift. */ @@ -163,15 +193,21 @@ const getBucketsForFilters = (filters, nodeTypeNames, typedKeyValueIndexes) => { query: { value: targetValue }, } = filter - ensureIndexByTypedChain(chain, nodeTypeNames, typedKeyValueIndexes) + let cacheKey = createTypedFilterCacheKey(nodeTypeNames, filter) - const nodesByKeyValue = getNodesByTypedChain( + ensureIndexByTypedChain( + cacheKey, chain, - targetValue, nodeTypeNames, typedKeyValueIndexes ) + const nodesByKeyValue = getNodesByTypedChain( + cacheKey, + targetValue, + typedKeyValueIndexes + ) + // If we couldn't find the needle then maybe sift can, for example if the // schema contained a proxy; `slug: String @proxy(from: "slugInternal")` // There are also cases (and tests) where id exists with a different type @@ -202,7 +238,7 @@ const getBucketsForFilters = (filters, nodeTypeNames, typedKeyValueIndexes) => { * @property {boolean} args.firstOnly true if you want to return only the first * result found. This will return a collection of size 1. Not a single element * @property {{filter?: Object, sort?: Object} | undefined} args.queryArgs - * @property {undefined | Map>>} args.typedKeyValueIndexes + * @property {undefined | Map>>} args.typedKeyValueIndexes * May be undefined. A cache of indexes where you can look up Nodes grouped * by a key: `types.join(',')+'/'+filterPath.join('+')`, which yields a Map * which holds a Set of Nodes for the value that the filter is trying to eq @@ -243,7 +279,7 @@ exports.runSift = runFilterAndSort * @param {Array | undefined} filterFields * @param {boolean} firstOnly * @param {Array} nodeTypeNames - * @param {undefined | Map>>} typedKeyValueIndexes + * @param {undefined | Map>>} typedKeyValueIndexes * @param resolvedFields * @returns {Array | undefined} Collection of results. Collection * will be limited to 1 if `firstOnly` is true @@ -318,7 +354,7 @@ const filterToStats = ( * * @param {Array} filters Resolved. (Should be checked by caller to exist) * @param {Array} nodeTypeNames - * @param {Map>>} typedKeyValueIndexes + * @param {Map>>} typedKeyValueIndexes * @returns {Array|undefined} Collection of results */ const filterWithoutSift = (filters, nodeTypeNames, typedKeyValueIndexes) => {