From 060b4ab6df8e063a4a814c8c5cc953347d93205d Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 22 Dec 2022 08:30:57 +0100 Subject: [PATCH] fix(gatsby): handle initializing multiple instances of gatsby-plugin-sharp (#37306) * fix(gatsby): handle initializing multiple instances of gatsby-plugin-sharp * fix(gatsby): handle initializing multiple instances of gatsby-plugin-sharp in engines * update standalone-regenerate * normalize main config when loading themes * move deduplicaiton to plugin loading instead of themes loading * convert load-themes to TS * lint * update assertions * remove test that no longer make sense (cherry picked from commit 26f2b72a63b29ef8425cd33ae70708279cd4af87) --- .../gatsby/src/bootstrap/load-config/index.ts | 2 +- .../load-plugins/__tests__/load-plugins.ts | 8 +- .../load-plugins/load-internal-plugins.ts | 5 +- .../bootstrap/load-themes/__tests__/index.js | 2 +- .../load-themes/{index.js => index.ts} | 146 ++++++++++++------ .../gatsby/src/schema/graphql-engine/entry.ts | 14 +- .../schema/graphql-engine/print-plugins.ts | 28 ++-- .../graphql-engine/standalone-regenerate.ts | 1 + .../utils/__tests__/merge-gatsby-config.ts | 61 -------- .../gatsby/src/utils/import-gatsby-plugin.ts | 11 +- .../gatsby/src/utils/merge-gatsby-config.ts | 41 +---- 11 files changed, 153 insertions(+), 166 deletions(-) rename packages/gatsby/src/bootstrap/load-themes/{index.js => index.ts} (57%) diff --git a/packages/gatsby/src/bootstrap/load-config/index.ts b/packages/gatsby/src/bootstrap/load-config/index.ts index 8d4e98ef2a0f4..585dfe8bc4a1f 100644 --- a/packages/gatsby/src/bootstrap/load-config/index.ts +++ b/packages/gatsby/src/bootstrap/load-config/index.ts @@ -3,7 +3,7 @@ import telemetry from "gatsby-telemetry" import { preferDefault } from "../prefer-default" import { getConfigFile } from "../get-config-file" import { internalActions } from "../../redux/actions" -import loadThemes from "../load-themes" +import { loadThemes } from "../load-themes" import { store } from "../../redux" import handleFlags from "../../utils/handle-flags" import availableFlags from "../../utils/flags" diff --git a/packages/gatsby/src/bootstrap/load-plugins/__tests__/load-plugins.ts b/packages/gatsby/src/bootstrap/load-plugins/__tests__/load-plugins.ts index c3e1afafed364..535b8bcccfb7d 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/__tests__/load-plugins.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/__tests__/load-plugins.ts @@ -218,9 +218,7 @@ describe(`Load plugins`, () => { (plugin: { name: string }) => plugin.name === `gatsby-plugin-typescript` ) - // TODO: I think we should probably be de-duping, so this should be 1. - // But this test is mostly here to ensure we don't add an _additional_ gatsby-plugin-typescript - expect(tsplugins.length).toEqual(2) + expect(tsplugins.length).toEqual(1) }) }) @@ -351,9 +349,7 @@ describe(`Load plugins`, () => { plugin.name === `gatsby-plugin-gatsby-cloud` ) - // TODO: I think we should probably be de-duping, so this should be 1. - // But this test is mostly here to ensure we don't add an _additional_ gatsby-plugin-typescript - expect(cloudPlugins.length).toEqual(2) + expect(cloudPlugins.length).toEqual(1) }) }) diff --git a/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts b/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts index e04783ee97f9b..e58a38e1deb64 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts @@ -1,4 +1,5 @@ import { slash } from "gatsby-core-utils" +import { uniqWith, isEqual } from "lodash" import path from "path" import reporter from "gatsby-cli/lib/reporter" import { store } from "../../redux" @@ -170,5 +171,7 @@ export function loadInternalPlugins( ) ) - return plugins + const uniquePlugins = uniqWith(plugins, isEqual) + + return uniquePlugins } diff --git a/packages/gatsby/src/bootstrap/load-themes/__tests__/index.js b/packages/gatsby/src/bootstrap/load-themes/__tests__/index.js index 68380baad20f8..026303262fade 100644 --- a/packages/gatsby/src/bootstrap/load-themes/__tests__/index.js +++ b/packages/gatsby/src/bootstrap/load-themes/__tests__/index.js @@ -1,4 +1,4 @@ -const loadThemes = require(`..`) +const { loadThemes } = require(`..`) const path = require(`path`) describe(`loadThemes`, () => { diff --git a/packages/gatsby/src/bootstrap/load-themes/index.js b/packages/gatsby/src/bootstrap/load-themes/index.ts similarity index 57% rename from packages/gatsby/src/bootstrap/load-themes/index.js rename to packages/gatsby/src/bootstrap/load-themes/index.ts index d38dfef3c1895..c853796e93b15 100644 --- a/packages/gatsby/src/bootstrap/load-themes/index.js +++ b/packages/gatsby/src/bootstrap/load-themes/index.ts @@ -1,22 +1,39 @@ -const { createRequireFromPath } = require(`gatsby-core-utils`) -const path = require(`path`) -import { mergeGatsbyConfig } from "../../utils/merge-gatsby-config" -const Promise = require(`bluebird`) -const _ = require(`lodash`) -const debug = require(`debug`)(`gatsby:load-themes`) +import { createRequireFromPath } from "gatsby-core-utils" +import * as path from "path" +import { + IGatsbyConfigInput, + mergeGatsbyConfig, + PluginEntry, + IPluginEntryWithParentDir, +} from "../../utils/merge-gatsby-config" +import { mapSeries } from "bluebird" +import { flattenDeep, isEqual, isFunction, uniqWith } from "lodash" +import DebugCtor from "debug" import { preferDefault } from "../prefer-default" import { getConfigFile } from "../get-config-file" import { resolvePlugin } from "../load-plugins/resolve-plugin" -const reporter = require(`gatsby-cli/lib/reporter`) +import reporter from "gatsby-cli/lib/reporter" + +const debug = DebugCtor(`gatsby:load-themes`) + +interface IThemeObj { + themeName: string + themeConfig: IGatsbyConfigInput + themeDir: string + themeSpec: PluginEntry + parentDir: string + configFilePath?: string +} // get the gatsby-config file for a theme const resolveTheme = async ( - themeSpec, - configFileThatDeclaredTheme, - isMainConfig = false, - rootDir -) => { - const themeName = themeSpec.resolve || themeSpec + themeSpec: PluginEntry, + configFileThatDeclaredTheme: string | undefined, + isMainConfig: boolean = false, + rootDir: string +): Promise => { + const themeName = + typeof themeSpec === `string` ? themeSpec : themeSpec.resolve let themeDir try { const scopedRequire = createRequireFromPath(`${rootDir}/:internal:`) @@ -59,13 +76,16 @@ const resolveTheme = async ( themeDir, `gatsby-config` ) - const theme = preferDefault(configModule) + const theme: + | IGatsbyConfigInput + | ((options?: Record) => IGatsbyConfigInput) = + preferDefault(configModule) // if theme is a function, call it with the themeConfig - let themeConfig = theme - if (_.isFunction(theme)) { - themeConfig = theme(themeSpec.options || {}) - } + const themeConfig = isFunction(theme) + ? theme(typeof themeSpec === `string` ? {} : themeSpec.options) + : theme + return { themeName, themeConfig, @@ -84,9 +104,9 @@ const resolveTheme = async ( // no use case for a loop so I expect that to only happen if someone is very // off track and creating their own set of themes const processTheme = ( - { themeName, themeConfig, themeSpec, themeDir, configFilePath }, - { rootDir } -) => { + { themeName, themeConfig, themeSpec, themeDir, configFilePath }: IThemeObj, + { rootDir }: { rootDir: string } +): Promise> => { const themesList = themeConfig && themeConfig.plugins // Gatsby themes don't have to specify a gatsby-config.js (they might only use gatsby-node, etc) // in this case they're technically plugins, but we should support it anyway @@ -94,24 +114,53 @@ const processTheme = ( if (themeConfig && themesList) { // for every parent theme a theme defines, resolve the parent's // gatsby config and return it in order [parentA, parentB, child] - return Promise.mapSeries(themesList, async spec => { - const themeObj = await resolveTheme(spec, configFilePath, false, themeDir) - return processTheme(themeObj, { rootDir: themeDir }) - }).then(arr => - arr.concat([ - { themeName, themeConfig, themeSpec, themeDir, parentDir: rootDir }, - ]) + return mapSeries( + themesList, + async (spec: PluginEntry): Promise> => { + const themeObj = await resolveTheme( + spec, + configFilePath, + false, + themeDir + ) + return processTheme(themeObj, { rootDir: themeDir }) + } + ).then(arr => + flattenDeep( + arr.concat([ + { themeName, themeConfig, themeSpec, themeDir, parentDir: rootDir }, + ]) + ) ) } else { // if a theme doesn't define additional themes, return the original theme - return [{ themeName, themeConfig, themeSpec, themeDir, parentDir: rootDir }] + return Promise.resolve([ + { themeName, themeConfig, themeSpec, themeDir, parentDir: rootDir }, + ]) } } -module.exports = async (config, { configFilePath, rootDir }) => { - const themesA = await Promise.mapSeries( +function normalizePluginEntry( + plugin: PluginEntry, + parentDir: string +): IPluginEntryWithParentDir { + return { + resolve: typeof plugin === `string` ? plugin : plugin.resolve, + options: typeof plugin === `string` ? {} : plugin.options || {}, + parentDir, + } +} + +export async function loadThemes( + config: IGatsbyConfigInput, + { configFilePath, rootDir }: { configFilePath: string; rootDir: string } +): Promise<{ + config: IGatsbyConfigInput + themes: Array +}> { + const themesA = await mapSeries( config.plugins || [], - async themeSpec => { + async (themeSpec: PluginEntry) => { const themeObj = await resolveTheme( themeSpec, configFilePath, @@ -120,7 +169,7 @@ module.exports = async (config, { configFilePath, rootDir }) => { ) return processTheme(themeObj, { rootDir }) } - ).then(arr => _.flattenDeep(arr)) + ).then(arr => flattenDeep(arr)) // log out flattened themes list to aid in debugging debug(themesA) @@ -129,21 +178,21 @@ module.exports = async (config, { configFilePath, rootDir }) => { // list in the config for the theme. This enables the usage of // gatsby-node, etc in themes. return ( - Promise.mapSeries( + mapSeries( themesA, ({ themeName, themeConfig = {}, themeSpec, themeDir, parentDir }) => { return { ...themeConfig, plugins: [ - ...(themeConfig.plugins || []).map(plugin => { - return { - resolve: typeof plugin === `string` ? plugin : plugin.resolve, - options: plugin.options || {}, - parentDir: themeDir, - } - }), + ...(themeConfig.plugins || []).map(plugin => + normalizePluginEntry(plugin, themeDir) + ), // theme plugin is last so it's gatsby-node, etc can override it's declared plugins, like a normal site. - { resolve: themeName, options: themeSpec.options || {}, parentDir }, + { + resolve: themeName, + options: typeof themeSpec === `string` ? {} : themeSpec.options, + parentDir, + }, ], } } @@ -156,8 +205,19 @@ module.exports = async (config, { configFilePath, rootDir }) => { */ .reduce(mergeGatsbyConfig, {}) .then(newConfig => { + const mergedConfig = mergeGatsbyConfig(newConfig, { + ...config, + plugins: [ + ...(config.plugins || []).map(plugin => + normalizePluginEntry(plugin, rootDir) + ), + ], + }) + + mergedConfig.plugins = uniqWith(mergedConfig.plugins, isEqual) + return { - config: mergeGatsbyConfig(newConfig, config), + config: mergedConfig, themes: themesA, } }) diff --git a/packages/gatsby/src/schema/graphql-engine/entry.ts b/packages/gatsby/src/schema/graphql-engine/entry.ts index c7383fbf0d0b6..f64a93df57561 100644 --- a/packages/gatsby/src/schema/graphql-engine/entry.ts +++ b/packages/gatsby/src/schema/graphql-engine/entry.ts @@ -59,18 +59,20 @@ export class GraphQLEngine { payload: flattenedPlugins, }) - for (const pluginName of Object.keys(gatsbyNodes)) { + for (const plugin of gatsbyNodes) { + const { name, module, importKey } = plugin setGatsbyPluginCache( - { name: pluginName, resolve: `` }, + { name, resolve: ``, importKey }, `gatsby-node`, - gatsbyNodes[pluginName] + module ) } - for (const pluginName of Object.keys(gatsbyWorkers)) { + for (const plugin of gatsbyWorkers) { + const { name, module, importKey } = plugin setGatsbyPluginCache( - { name: pluginName, resolve: `` }, + { name, resolve: ``, importKey }, `gatsby-worker`, - gatsbyWorkers[pluginName] + module ) } diff --git a/packages/gatsby/src/schema/graphql-engine/print-plugins.ts b/packages/gatsby/src/schema/graphql-engine/print-plugins.ts index 2019e93184f6f..568837aeca615 100644 --- a/packages/gatsby/src/schema/graphql-engine/print-plugins.ts +++ b/packages/gatsby/src/schema/graphql-engine/print-plugins.ts @@ -53,24 +53,24 @@ async function render( usedPlugins: IGatsbyState["flattenedPlugins"], usedSubPlugins: IGatsbyState["flattenedPlugins"] ): Promise { - const uniqGatsbyNode = uniq(usedPlugins) const uniqSubPlugins = uniq(usedSubPlugins) - const sanitizedUsedPlugins = usedPlugins.map(plugin => { + const sanitizedUsedPlugins = usedPlugins.map((plugin, i) => { // TODO: We don't support functions in pluginOptions here return { ...plugin, resolve: ``, pluginFilepath: ``, subPluginPaths: undefined, + importKey: i + 1, } }) - const pluginsWithWorkers = await filterPluginsWithWorkers(uniqGatsbyNode) + const pluginsWithWorkers = await filterPluginsWithWorkers(usedPlugins) const subPluginModuleToImportNameMapping = new Map() const imports: Array = [ - ...uniqGatsbyNode.map( + ...usedPlugins.map( (plugin, i) => `import * as pluginGatsbyNode${i} from "${relativePluginPath( plugin.resolve @@ -90,22 +90,28 @@ async function render( )}"` }), ] - const gatsbyNodeExports = uniqGatsbyNode.map( - (plugin, i) => `"${plugin.name}": pluginGatsbyNode${i},` + const gatsbyNodeExports = usedPlugins.map( + (plugin, i) => + `{ name: "${plugin.name}", module: pluginGatsbyNode${i}, importKey: ${ + i + 1 + } },` ) const gatsbyWorkerExports = pluginsWithWorkers.map( - (plugin, i) => `"${plugin.name}": pluginGatsbyWorker${i},` + (plugin, i) => + `{ name: "${plugin.name}", module: pluginGatsbyWorker${i}, importKey: ${ + i + 1 + } },` ) const output = ` ${imports.join(`\n`)} -export const gatsbyNodes = { +export const gatsbyNodes = [ ${gatsbyNodeExports.join(`\n`)} -} +] -export const gatsbyWorkers = { +export const gatsbyWorkers = [ ${gatsbyWorkerExports.join(`\n`)} -} +] export const flattenedPlugins = ${JSON.stringify( diff --git a/packages/gatsby/src/schema/graphql-engine/standalone-regenerate.ts b/packages/gatsby/src/schema/graphql-engine/standalone-regenerate.ts index 628725bf98508..a76ef7bc14a96 100644 --- a/packages/gatsby/src/schema/graphql-engine/standalone-regenerate.ts +++ b/packages/gatsby/src/schema/graphql-engine/standalone-regenerate.ts @@ -26,6 +26,7 @@ import { store } from "../../redux" import { validateEngines } from "../../utils/validate-engines" async function run(): Promise { + process.env.GATSBY_SLICES = `1` // load config console.log(`loading config and plugins`) await loadConfigAndPlugins({ diff --git a/packages/gatsby/src/utils/__tests__/merge-gatsby-config.ts b/packages/gatsby/src/utils/__tests__/merge-gatsby-config.ts index 9fe5959f5b17d..f16f217c35638 100644 --- a/packages/gatsby/src/utils/__tests__/merge-gatsby-config.ts +++ b/packages/gatsby/src/utils/__tests__/merge-gatsby-config.ts @@ -36,67 +36,6 @@ describe(`Merge gatsby config`, () => { }) }) - it(`Merging plugins uniqs them, keeping the first occurrence`, () => { - const basicConfig = { - plugins: [ - `gatsby-plugin-mdx`, - { - resolve: `scoped-plugin`, - options: {}, - parentDir: `/path/to/scoped-basic/parent`, - }, - ], - } - const morePlugins = { - plugins: [ - `a-plugin`, - `gatsby-plugin-mdx`, - `b-plugin`, - { - resolve: `c-plugin`, - options: {}, - }, - { - resolve: `scoped-plugin`, - options: {}, - parentDir: `/path/to/scoped-more/parent`, - }, - ], - } - expect(mergeGatsbyConfig(basicConfig, morePlugins)).toEqual({ - plugins: [ - `gatsby-plugin-mdx`, - { - resolve: `scoped-plugin`, - options: {}, - parentDir: `/path/to/scoped-basic/parent`, - }, - `a-plugin`, - `b-plugin`, - { - resolve: `c-plugin`, - options: {}, - }, - ], - }) - expect(mergeGatsbyConfig(morePlugins, basicConfig)).toEqual({ - plugins: [ - `a-plugin`, - `gatsby-plugin-mdx`, - `b-plugin`, - { - resolve: `c-plugin`, - options: {}, - }, - { - resolve: `scoped-plugin`, - options: {}, - parentDir: `/path/to/scoped-more/parent`, - }, - ], - }) - }) - it(`Merging siteMetadata is recursive`, () => { const a = { siteMetadata: { diff --git a/packages/gatsby/src/utils/import-gatsby-plugin.ts b/packages/gatsby/src/utils/import-gatsby-plugin.ts index 6c4ef347ad9c9..fc5ef978a4c86 100644 --- a/packages/gatsby/src/utils/import-gatsby-plugin.ts +++ b/packages/gatsby/src/utils/import-gatsby-plugin.ts @@ -7,12 +7,18 @@ import { preferDefault } from "../bootstrap/prefer-default" const pluginModuleCache = new Map() export function setGatsbyPluginCache( - plugin: { name: string; resolve: string }, + plugin: { name: string; resolve: string; importKey?: string }, module: string, moduleObject: any ): void { const key = `${plugin.name}/${module}` pluginModuleCache.set(key, moduleObject) + + const additionalPrefix = plugin.importKey || plugin.resolve + if (additionalPrefix) { + const key = `${additionalPrefix}/${module}` + pluginModuleCache.set(key, moduleObject) + } } export async function importGatsbyPlugin( @@ -20,10 +26,11 @@ export async function importGatsbyPlugin( name: string resolve: string resolvedCompiledGatsbyNode?: string + importKey?: string }, module: string ): Promise { - const key = `${plugin.name}/${module}` + const key = `${plugin.importKey || plugin.resolve || plugin.name}/${module}` let pluginModule = pluginModuleCache.get(key) diff --git a/packages/gatsby/src/utils/merge-gatsby-config.ts b/packages/gatsby/src/utils/merge-gatsby-config.ts index ba765a9d105b3..cd755e45c26ea 100644 --- a/packages/gatsby/src/utils/merge-gatsby-config.ts +++ b/packages/gatsby/src/utils/merge-gatsby-config.ts @@ -1,20 +1,16 @@ import _ from "lodash" import { Express } from "express" import type { TrailingSlash } from "gatsby-page-utils" -// TODO export it in index.d.ts -type PluginEntry = - | string - | { - resolve: string - options?: Record - } -interface INormalizedPluginEntry { +export interface IPluginEntryWithParentDir { resolve: string - options: Record + options?: Record + parentDir: string } +// TODO export it in index.d.ts +export type PluginEntry = string | IPluginEntryWithParentDir -interface IGatsbyConfigInput { +export interface IGatsbyConfigInput { siteMetadata?: Record plugins?: Array pathPrefix?: string @@ -35,23 +31,6 @@ type ConfigKey = keyof IGatsbyConfigInput type Metadata = IGatsbyConfigInput["siteMetadata"] type Mapping = IGatsbyConfigInput["mapping"] -/** - * Normalize plugin spec before comparing so - * - `gatsby-plugin-name` - * - { resolve: `gatsby-plugin-name` } - * - { resolve: `gatsby-plugin-name`, options: {} } - * are all considered equal - */ -const normalizePluginEntry = (entry: PluginEntry): INormalizedPluginEntry => - _.isString(entry) - ? { - resolve: entry, - options: {}, - } - : _.isObject(entry) - ? { options: {}, ...entry } - : entry - const howToMerge = { /** * pick a truthy value by default. @@ -65,13 +44,7 @@ const howToMerge = { plugins: ( a: Array = [], b: Array = [] - ): Array => - _.uniqWith(a.concat(b), (a, b) => - _.isEqual( - _.pick(normalizePluginEntry(a), [`resolve`, `options`]), - _.pick(normalizePluginEntry(b), [`resolve`, `options`]) - ) - ), + ): Array => a.concat(b), mapping: (objA: Mapping, objB: Mapping): Mapping => _.merge({}, objA, objB), } as const