From 4f5bbda374c63d55ec6d956cb7e0c3e7620e1d10 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Sat, 5 Jan 2019 14:06:04 +0100 Subject: [PATCH 01/36] feat(sharp): thumbnail creation on demand in dev mode --- packages/gatsby-plugin-sharp/.gitignore | 3 + packages/gatsby-plugin-sharp/src/cache.js | 3 + .../gatsby-plugin-sharp/src/gatsby-node.js | 17 ++ packages/gatsby-plugin-sharp/src/index.js | 262 ++++-------------- .../gatsby-plugin-sharp/src/processFile.js | 177 ++++++++++++ 5 files changed, 248 insertions(+), 214 deletions(-) create mode 100644 packages/gatsby-plugin-sharp/src/cache.js create mode 100644 packages/gatsby-plugin-sharp/src/processFile.js diff --git a/packages/gatsby-plugin-sharp/.gitignore b/packages/gatsby-plugin-sharp/.gitignore index 69f486d61b768..2d94dfe09b132 100644 --- a/packages/gatsby-plugin-sharp/.gitignore +++ b/packages/gatsby-plugin-sharp/.gitignore @@ -2,3 +2,6 @@ /node_modules /gatsby-node.js /duotone.js +/cache.js +/queue.js +/processFile.js diff --git a/packages/gatsby-plugin-sharp/src/cache.js b/packages/gatsby-plugin-sharp/src/cache.js new file mode 100644 index 0000000000000..f94d2d7bc18a9 --- /dev/null +++ b/packages/gatsby-plugin-sharp/src/cache.js @@ -0,0 +1,3 @@ +const cache = new Map() + +module.exports = cache diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 66c14a6f39f81..04f3faf0b7678 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -1,10 +1,27 @@ const { setBoundActionCreators, setPluginOptions } = require(`./index`) +const cache = require(`./cache`) +const processFile = require(`./processFile`) exports.onPreInit = ({ actions }, pluginOptions) => { setBoundActionCreators(actions) setPluginOptions(pluginOptions) } +exports.onCreateDevServer = ({ app }) => { + app.use((req, res, next) => { + if (cache.has(req.originalUrl)) { + const { job, pluginOptions } = cache.get(req.originalUrl) + return Promise.all( + processFile(job.file.absolutePath, [job], pluginOptions) + ).then(() => { + res.sendFile(job.outputPath) + }) + } + + return next() + }) +} + // TODO // exports.formatJobMessage = jobs => { // return { diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index 62b74f04f4442..2b261881353db 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -2,16 +2,13 @@ const sharp = require(`sharp`) const crypto = require(`crypto`) const imageSize = require(`probe-image-size`) const _ = require(`lodash`) -const Promise = require(`bluebird`) const fs = require(`fs-extra`) const ProgressBar = require(`progress`) -const imagemin = require(`imagemin`) -const imageminMozjpeg = require(`imagemin-mozjpeg`) -const imageminPngquant = require(`imagemin-pngquant`) -const imageminWebp = require(`imagemin-webp`) const queue = require(`async/queue`) const path = require(`path`) const existsSync = require(`fs-exists-cached`).sync +const cache = require(`./cache`) +const processFile = require(`./processFile`) const imageSizeCache = new Map() const getImageSize = file => { @@ -51,16 +48,6 @@ exports.setPluginOptions = opts => { pluginOptions = Object.assign({}, pluginOptions, opts) } -// Promisify the sharp prototype (methods) to promisify the alternative (for -// raw) callback-accepting toBuffer(...) method -Promise.promisifyAll(sharp.prototype, { multiArgs: true }) - -// Try to enable the use of SIMD instructions. Seems to provide a smallish -// speedup on resizing heavy loads (~10%). Sharp disables this feature by -// default as there's been problems with segfaulting in the past but we'll be -// adventurous and see what happens with it on. -sharp.simd(true) - const bar = new ProgressBar( `Generating image thumbnails [:bar] :current/:total :elapsed secs :percent`, { @@ -124,192 +111,6 @@ const healOptions = (args, defaultArgs) => { } let totalJobs = 0 -const processFile = (file, jobs, cb, reporter) => { - // console.log("totalJobs", totalJobs) - bar.total = totalJobs - - let imagesFinished = 0 - - // Wait for each job promise to resolve. - Promise.all(jobs.map(job => job.finishedPromise)).then(() => cb()) - - let pipeline - try { - pipeline = sharp(file) - - // Keep Metadata - if (!pluginOptions.stripMetadata) { - pipeline = pipeline.withMetadata() - } - - pipeline = pipeline.rotate() - } catch (err) { - reportError(`Failed to process image ${file}`, err, reporter) - jobs.forEach(job => job.outsideReject(err)) - return - } - - jobs.forEach(async job => { - const args = job.args - let clonedPipeline - if (jobs.length > 1) { - clonedPipeline = pipeline.clone() - } else { - clonedPipeline = pipeline - } - // Sharp only allows ints as height/width. Since both aren't always - // set, check first before trying to round them. - let roundedHeight = args.height - if (roundedHeight) { - roundedHeight = Math.round(roundedHeight) - } - - let roundedWidth = args.width - if (roundedWidth) { - roundedWidth = Math.round(roundedWidth) - } - - clonedPipeline - .resize(roundedWidth, roundedHeight, { - position: args.cropFocus, - }) - .png({ - compressionLevel: args.pngCompressionLevel, - adaptiveFiltering: false, - force: args.toFormat === `png`, - }) - .webp({ - quality: args.quality, - force: args.toFormat === `webp`, - }) - .tiff({ - quality: args.quality, - force: args.toFormat === `tiff`, - }) - - // jpeg - if (!pluginOptions.useMozJpeg) { - clonedPipeline = clonedPipeline.jpeg({ - quality: args.quality, - progressive: args.jpegProgressive, - force: args.toFormat === `jpg`, - }) - } - - // grayscale - if (args.grayscale) { - clonedPipeline = clonedPipeline.grayscale() - } - - // rotate - if (args.rotate && args.rotate !== 0) { - clonedPipeline = clonedPipeline.rotate(args.rotate) - } - - // duotone - if (args.duotone) { - clonedPipeline = await duotone( - args.duotone, - args.toFormat || job.file.extension, - clonedPipeline - ) - } - - const onFinish = err => { - imagesFinished += 1 - bar.tick() - boundActionCreators.setJob( - { - id: `processing image ${job.file.absolutePath}`, - imagesFinished, - }, - { name: `gatsby-plugin-sharp` } - ) - - if (err) { - reportError(`Failed to process image ${file}`, err, reporter) - job.outsideReject(err) - } else { - job.outsideResolve() - } - } - if ( - (job.file.extension === `png` && args.toFormat === ``) || - args.toFormat === `png` - ) { - clonedPipeline - .toBuffer() - .then(sharpBuffer => - imagemin - .buffer(sharpBuffer, { - plugins: [ - imageminPngquant({ - quality: `${args.quality}-${Math.min( - args.quality + 25, - 100 - )}`, - speed: args.pngCompressionSpeed - ? args.pngCompressionSpeed - : undefined, - strip: !!pluginOptions.stripMetadata, // Must be a bool - }), - ], - }) - .then(imageminBuffer => { - fs.writeFile(job.outputPath, imageminBuffer, onFinish) - }) - .catch(onFinish) - ) - .catch(onFinish) - // Compress jpeg - } else if ( - pluginOptions.useMozJpeg && - ((job.file.extension === `jpg` && args.toFormat === ``) || - (job.file.extension === `jpeg` && args.toFormat === ``) || - args.toFormat === `jpg`) - ) { - clonedPipeline - .toBuffer() - .then(sharpBuffer => - imagemin - .buffer(sharpBuffer, { - plugins: [ - imageminMozjpeg({ - quality: args.quality, - progressive: args.jpegProgressive, - }), - ], - }) - .then(imageminBuffer => { - fs.writeFile(job.outputPath, imageminBuffer, onFinish) - }) - .catch(onFinish) - ) - .catch(onFinish) - // Compress webp - } else if ( - (job.file.extension === `webp` && args.toFormat === ``) || - args.toFormat === `webp` - ) { - clonedPipeline - .toBuffer() - .then(sharpBuffer => - imagemin - .buffer(sharpBuffer, { - plugins: [imageminWebp({ quality: args.quality })], - }) - .then(imageminBuffer => { - fs.writeFile(job.outputPath, imageminBuffer, onFinish) - }) - .catch(onFinish) - ) - .catch(onFinish) - // any other format (tiff) - don't compress it just handle output - } else { - clonedPipeline.toFile(job.outputPath, onFinish) - } - }) -} const toProcess = {} const q = queue((task, callback) => { @@ -353,20 +154,45 @@ const queueJob = (job, reporter) => { { name: `gatsby-plugin-sharp` } ) // We're now processing the file's jobs. - processFile( + let imagesFinished = 0 + bar.total = totalJobs + let promises = processFile( job.file.absolutePath, jobs, - () => { - boundActionCreators.endJob( - { - id: `processing image ${job.file.absolutePath}`, - }, - { name: `gatsby-plugin-sharp` } - ) - cb() - }, - reporter + pluginOptions + ).map(promise => + promise + .then(() => { + job.outsideResolve() + }) + .catch(err => { + reportError( + `Failed to process image ${job.file.absolutePath}`, + err, + reporter + ) + job.outsideReject(err) + }) + .then(() => { + imagesFinished += 1 + bar.tick() + boundActionCreators.setJob( + { + id: `processing image ${job.file.absolutePath}`, + imagesFinished, + }, + { name: `gatsby-plugin-sharp` } + ) + }) ) + + Promise.all(promises).then(() => { + boundActionCreators.endJob( + { id: `processing image ${job.file.absolutePath}` }, + { name: `gatsby-plugin-sharp` } + ) + cb() + }) }) } } @@ -454,7 +280,9 @@ function queueImageResizing({ file, args = {}, reporter }) { outputPath: filePath, } - queueJob(job, reporter) + if (process.env.NODE_ENV === `production`) { + queueJob(job, reporter) + } // encode the file name for URL const encodedImgSrc = `/${encodeURIComponent(file.name)}.${fileExtension}` @@ -464,6 +292,10 @@ function queueImageResizing({ file, args = {}, reporter }) { const prefixedSrc = options.pathPrefix + `/static/${digestDirPrefix}` + encodedImgSrc + if (process.env.NODE_ENV !== `production`) { + cache.set(prefixedSrc, { job, pluginOptions }) + } + return { src: prefixedSrc, absolutePath: filePath, @@ -518,7 +350,9 @@ async function generateBase64({ file, args, reporter }) { pipeline ) } - const [buffer, info] = await pipeline.toBufferAsync() + const { data: buffer, info } = await pipeline.toBuffer({ + resolveWithObject: true, + }) const base64output = { src: `data:image/${info.format};base64,${buffer.toString(`base64`)}`, width: info.width, diff --git a/packages/gatsby-plugin-sharp/src/processFile.js b/packages/gatsby-plugin-sharp/src/processFile.js new file mode 100644 index 0000000000000..02b153937b976 --- /dev/null +++ b/packages/gatsby-plugin-sharp/src/processFile.js @@ -0,0 +1,177 @@ +const sharp = require(`sharp`) +const fs = require(`fs-extra`) +const debug = require(`debug`)(`gatsby:gatsby-plugin-sharp`) +const duotone = require(`./duotone`) +const imagemin = require(`imagemin`) +const imageminMozjpeg = require(`imagemin-mozjpeg`) +const imageminPngquant = require(`imagemin-pngquant`) +const imageminWebp = require(`imagemin-webp`) + +const writeFileAsync = (file, buffer) => + new Promise((resovle, reject) => { + fs.writeFile(file, buffer, err => { + if (err) { + return reject(err) + } + + return resovle() + }) + }) + +// Try to enable the use of SIMD instructions. Seems to provide a smallish +// speedup on resizing heavy loads (~10%). Sharp disables this feature by +// default as there's been problems with segfaulting in the past but we'll be +// adventurous and see what happens with it on. +sharp.simd(true) + +module.exports = (file, transforms, options = {}) => { + let pipeline + try { + pipeline = sharp(file) + + // Keep Metadata + if (!options.stripMetadata) { + pipeline = pipeline.withMetadata() + } + + pipeline = pipeline.rotate() + } catch (err) { + throw new Error(`Failed to process image ${file}`) + } + + return transforms.map(async ({ file: transformFile, outputPath, args }) => { + debug(`Start processing ${outputPath}`) + + let clonedPipeline = transforms.length > 1 ? pipeline.clone() : pipeline + + // Sharp only allows ints as height/width. Since both aren't always + // set, check first before trying to round them. + let roundedHeight = args.height + if (roundedHeight) { + roundedHeight = Math.round(roundedHeight) + } + + let roundedWidth = args.width + if (roundedWidth) { + roundedWidth = Math.round(roundedWidth) + } + + clonedPipeline + .resize(roundedWidth, roundedHeight, { + position: args.cropFocus, + }) + .png({ + compressionLevel: args.pngCompressionLevel, + adaptiveFiltering: false, + force: args.toFormat === `png`, + }) + .webp({ + quality: args.quality, + force: args.toFormat === `webp`, + }) + .tiff({ + quality: args.quality, + force: args.toFormat === `tiff`, + }) + + // jpeg + if (!options.useMozJpeg) { + clonedPipeline = clonedPipeline.jpeg({ + quality: args.quality, + progressive: args.jpegProgressive, + force: args.toFormat === `jpg`, + }) + } + + // grayscale + if (args.grayscale) { + clonedPipeline = clonedPipeline.grayscale() + } + + // rotate + if (args.rotate && args.rotate !== 0) { + clonedPipeline = clonedPipeline.rotate(args.rotate) + } + + // duotone + if (args.duotone) { + clonedPipeline = await duotone( + args.duotone, + args.toFormat || transformFile.extension, + clonedPipeline + ) + } + + // lets decide how we want to save this transform + if ( + (transformFile.extension === `png` && args.toFormat === ``) || + args.toFormat === `png` + ) { + return compressPng(clonedPipeline, outputPath, { + ...args, + stripMetadata: options.stripMetadata, + }) + } + + if ( + options.useMozJpeg && + ((transformFile.extension === `jpg` && args.toFormat === ``) || + (transformFile.extension === `jpeg` && args.toFormat === ``) || + args.toFormat === `jpg`) + ) { + return compressJpg(clonedPipeline, outputPath, args) + } + + if ( + (file.extension === `webp` && args.toFormat === ``) || + args.toFormat === `webp` + ) { + return compressWebP(clonedPipeline, outputPath, args) + } + + return clonedPipeline.toFile(outputPath) + }) +} + +const compressPng = (pipeline, outputPath, options) => + pipeline.toBuffer().then(sharpBuffer => + imagemin + .buffer(sharpBuffer, { + plugins: [ + imageminPngquant({ + quality: `${options.quality}-${Math.min( + options.quality + 25, + 100 + )}`, // e.g. 40-65 + speed: options.pngCompressionSpeed + ? options.pngCompressionSpeed + : undefined, + strip: !!options.stripMetadata, // Must be a bool + }), + ], + }) + .then(imageminBuffer => writeFileAsync(outputPath, imageminBuffer)) + ) + +const compressJpg = (pipeline, outputPath, options) => + pipeline.toBuffer().then(sharpBuffer => + imagemin + .buffer(sharpBuffer, { + plugins: [ + imageminMozjpeg({ + quality: options.quality, + progressive: options.jpegProgressive, + }), + ], + }) + .then(imageminBuffer => writeFileAsync(outputPath, imageminBuffer)) + ) + +const compressWebP = (pipeline, outputPath, options) => + pipeline.toBuffer().then(sharpBuffer => + imagemin + .buffer(sharpBuffer, { + plugins: [imageminWebp({ quality: options.quality })], + }) + .then(imageminBuffer => writeFileAsync(outputPath, imageminBuffer)) + ) From 432eb0259592550512e9903dbb1eb24458d024f1 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 10 Jan 2019 10:38:24 +0100 Subject: [PATCH 02/36] Fix thumbnail generation on gatsby build --- packages/gatsby-plugin-sharp/.gitignore | 2 +- packages/gatsby-plugin-sharp/src/cache.js | 3 - .../gatsby-plugin-sharp/src/gatsby-node.js | 67 ++++++-- packages/gatsby-plugin-sharp/src/index.js | 144 +++--------------- .../gatsby-plugin-sharp/src/processFile.js | 15 +- packages/gatsby-plugin-sharp/src/scheduler.js | 128 ++++++++++++++++ 6 files changed, 212 insertions(+), 147 deletions(-) delete mode 100644 packages/gatsby-plugin-sharp/src/cache.js create mode 100644 packages/gatsby-plugin-sharp/src/scheduler.js diff --git a/packages/gatsby-plugin-sharp/.gitignore b/packages/gatsby-plugin-sharp/.gitignore index 2d94dfe09b132..a331b6826346d 100644 --- a/packages/gatsby-plugin-sharp/.gitignore +++ b/packages/gatsby-plugin-sharp/.gitignore @@ -2,6 +2,6 @@ /node_modules /gatsby-node.js /duotone.js -/cache.js /queue.js /processFile.js +/scheduler.js diff --git a/packages/gatsby-plugin-sharp/src/cache.js b/packages/gatsby-plugin-sharp/src/cache.js deleted file mode 100644 index f94d2d7bc18a9..0000000000000 --- a/packages/gatsby-plugin-sharp/src/cache.js +++ /dev/null @@ -1,3 +0,0 @@ -const cache = new Map() - -module.exports = cache diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 04f3faf0b7678..559da0a40ffe6 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -1,24 +1,65 @@ -const { setBoundActionCreators, setPluginOptions } = require(`./index`) -const cache = require(`./cache`) +const { + setBoundActionCreators, + setPluginOptions, + queue, + reportError, +} = require(`./index`) const processFile = require(`./processFile`) +const { scheduleJob } = require(`./scheduler`) + +exports.onPreInit = ({ actions, cache }, pluginOptions) => { + cache.init() -exports.onPreInit = ({ actions }, pluginOptions) => { setBoundActionCreators(actions) setPluginOptions(pluginOptions) } -exports.onCreateDevServer = ({ app }) => { - app.use((req, res, next) => { - if (cache.has(req.originalUrl)) { - const { job, pluginOptions } = cache.get(req.originalUrl) - return Promise.all( - processFile(job.file.absolutePath, [job], pluginOptions) - ).then(() => { - res.sendFile(job.outputPath) - }) +exports.onPostBootstrap = async ({ cache }) => { + // JSON.stringify doesn't work on an Map so we need to convert it to an array + return cache.set(`queue`, Array.from(queue)) +} + +/** + * Execute all unprocessed images on gatsby build + */ +exports.onPostBuild = async ( + { cache, boundActionCreators, reporter }, + pluginOptions +) => { + const rawQueue = await cache.get(`queue`) + const queue = new Map(rawQueue) + + let promises = [] + for (const [, job] of queue) { + promises.push(scheduleJob(job, boundActionCreators, pluginOptions)) + } + + return Promise.all(promises).catch(({ err, message }) => { + reportError(message || err.message, err, reporter) + }) +} + +/** + * Build images on the fly when they are requested by the browser + */ +exports.onCreateDevServer = ({ app, cache }, pluginOptions) => { + app.use(async (req, res, next) => { + const rawQueue = await cache.get(`queue`) + if (!rawQueue) { + return next() + } + + const queue = new Map(rawQueue) + if (!queue.has(req.originalUrl)) { + return next() } - return next() + const job = queue.get(req.originalUrl) + + // wait until the file has been processed and saved to disk + await Promise.all(processFile(job.file.absolutePath, [job], pluginOptions)) + + return res.sendFile(job.outputPath) }) } diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index 2b261881353db..97f295c640c81 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -3,12 +3,8 @@ const crypto = require(`crypto`) const imageSize = require(`probe-image-size`) const _ = require(`lodash`) const fs = require(`fs-extra`) -const ProgressBar = require(`progress`) -const queue = require(`async/queue`) const path = require(`path`) -const existsSync = require(`fs-exists-cached`).sync -const cache = require(`./cache`) -const processFile = require(`./processFile`) +const { scheduleJob } = require(`./scheduler`) const imageSizeCache = new Map() const getImageSize = file => { @@ -38,6 +34,10 @@ exports.setBoundActionCreators = actions => { boundActionCreators = actions } +// We set the queue to a Map instead of an array to easily search in onCreateDevServer Api hook +const queue = new Map() +exports.queue = queue + /// Plugin options are loaded onPreInit in gatsby-node const pluginDefaults = { useMozJpeg: process.env.GATSBY_JPEG_ENCODER === `MOZJPEG`, @@ -48,14 +48,6 @@ exports.setPluginOptions = opts => { pluginOptions = Object.assign({}, pluginOptions, opts) } -const bar = new ProgressBar( - `Generating image thumbnails [:bar] :current/:total :elapsed secs :percent`, - { - total: 0, - width: 30, - } -) - const reportError = (message, err, reporter) => { if (reporter) { reporter.error(message, err) @@ -67,6 +59,7 @@ const reportError = (message, err, reporter) => { process.exit(1) } } +exports.reportError = reportError const generalArgs = { quality: 50, @@ -110,93 +103,6 @@ const healOptions = (args, defaultArgs) => { return options } -let totalJobs = 0 - -const toProcess = {} -const q = queue((task, callback) => { - task(callback) -}, 1) - -const queueJob = (job, reporter) => { - const inputFileKey = job.file.absolutePath.replace(/\./g, `%2E`) - const outputFileKey = job.outputPath.replace(/\./g, `%2E`) - const jobPath = `${inputFileKey}.${outputFileKey}` - - // Check if the job has already been queued. If it has, there's nothing - // to do, return. - if (_.has(toProcess, jobPath)) { - return - } - - // Check if the output file already exists so we don't redo work. - if (existsSync(job.outputPath)) { - return - } - - let notQueued = true - if (toProcess[inputFileKey]) { - notQueued = false - } - _.set(toProcess, jobPath, job) - - totalJobs += 1 - - if (notQueued) { - q.push(cb => { - const jobs = _.values(toProcess[inputFileKey]) - // Delete the input key from the toProcess list so more jobs can be queued. - delete toProcess[inputFileKey] - boundActionCreators.createJob( - { - id: `processing image ${job.file.absolutePath}`, - imagesCount: _.values(toProcess[inputFileKey]).length, - }, - { name: `gatsby-plugin-sharp` } - ) - // We're now processing the file's jobs. - let imagesFinished = 0 - bar.total = totalJobs - let promises = processFile( - job.file.absolutePath, - jobs, - pluginOptions - ).map(promise => - promise - .then(() => { - job.outsideResolve() - }) - .catch(err => { - reportError( - `Failed to process image ${job.file.absolutePath}`, - err, - reporter - ) - job.outsideReject(err) - }) - .then(() => { - imagesFinished += 1 - bar.tick() - boundActionCreators.setJob( - { - id: `processing image ${job.file.absolutePath}`, - imagesFinished, - }, - { name: `gatsby-plugin-sharp` } - ) - }) - ) - - Promise.all(promises).then(() => { - boundActionCreators.endJob( - { id: `processing image ${job.file.absolutePath}` }, - { name: `gatsby-plugin-sharp` } - ) - cb() - }) - }) - } -} - function queueImageResizing({ file, args = {}, reporter }) { const options = healOptions(args, {}) // Filter out false args, and args not for this extension and put width at @@ -236,13 +142,6 @@ function queueImageResizing({ file, args = {}, reporter }) { const filePath = path.join(dirPath, imgSrc) fs.ensureDirSync(dirPath) - // Create function to call when the image is finished. - let outsideResolve, outsideReject - const finishedPromise = new Promise((resolve, reject) => { - outsideResolve = resolve - outsideReject = reject - }) - let width let height // Calculate the eventual width/height of the image. @@ -269,21 +168,6 @@ function queueImageResizing({ file, args = {}, reporter }) { width = Math.round(options.height * aspectRatio) } - // Create job and process. - const job = { - file, - args: options, - finishedPromise, - outsideResolve, - outsideReject, - inputPath: file.absolutePath, - outputPath: filePath, - } - - if (process.env.NODE_ENV === `production`) { - queueJob(job, reporter) - } - // encode the file name for URL const encodedImgSrc = `/${encodeURIComponent(file.name)}.${fileExtension}` @@ -292,17 +176,27 @@ function queueImageResizing({ file, args = {}, reporter }) { const prefixedSrc = options.pathPrefix + `/static/${digestDirPrefix}` + encodedImgSrc - if (process.env.NODE_ENV !== `production`) { - cache.set(prefixedSrc, { job, pluginOptions }) + // Create job and add it to the queue, the queue will be processed inside gatsby-node.js + const job = { + file, + args: options, + inputPath: file.absolutePath, + outputPath: filePath, + src: prefixedSrc, } + queue.set(prefixedSrc, job) + return { src: prefixedSrc, absolutePath: filePath, width, height, aspectRatio, - finishedPromise, + // finishedPromise is needed to not break our API (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-transformer-sqip/src/extend-node-type.js#L115) + finishedPromise: () => ({ + then: () => scheduleJob(job, boundActionCreators, pluginOptions), + }), originalName: originalName, } } diff --git a/packages/gatsby-plugin-sharp/src/processFile.js b/packages/gatsby-plugin-sharp/src/processFile.js index 02b153937b976..f4ee6c7449b62 100644 --- a/packages/gatsby-plugin-sharp/src/processFile.js +++ b/packages/gatsby-plugin-sharp/src/processFile.js @@ -39,7 +39,8 @@ module.exports = (file, transforms, options = {}) => { throw new Error(`Failed to process image ${file}`) } - return transforms.map(async ({ file: transformFile, outputPath, args }) => { + return transforms.map(async transform => { + const { file: transformFile, outputPath, args } = transform debug(`Start processing ${outputPath}`) let clonedPipeline = transforms.length > 1 ? pipeline.clone() : pipeline @@ -107,10 +108,11 @@ module.exports = (file, transforms, options = {}) => { (transformFile.extension === `png` && args.toFormat === ``) || args.toFormat === `png` ) { - return compressPng(clonedPipeline, outputPath, { + await compressPng(clonedPipeline, outputPath, { ...args, stripMetadata: options.stripMetadata, }) + return transform } if ( @@ -119,17 +121,20 @@ module.exports = (file, transforms, options = {}) => { (transformFile.extension === `jpeg` && args.toFormat === ``) || args.toFormat === `jpg`) ) { - return compressJpg(clonedPipeline, outputPath, args) + await compressJpg(clonedPipeline, outputPath, args) + return transform } if ( (file.extension === `webp` && args.toFormat === ``) || args.toFormat === `webp` ) { - return compressWebP(clonedPipeline, outputPath, args) + await compressWebP(clonedPipeline, outputPath, args) + return transform } - return clonedPipeline.toFile(outputPath) + await clonedPipeline.toFile(outputPath) + return transform }) } diff --git a/packages/gatsby-plugin-sharp/src/scheduler.js b/packages/gatsby-plugin-sharp/src/scheduler.js new file mode 100644 index 0000000000000..3c05d21a10ef5 --- /dev/null +++ b/packages/gatsby-plugin-sharp/src/scheduler.js @@ -0,0 +1,128 @@ +const _ = require("lodash") +const ProgressBar = require(`progress`) +const existsSync = require(`fs-exists-cached`).sync +const queue = require(`async/queue`) +const processFile = require("./processFile") + +const toProcess = {} +let totalJobs = 0 +const q = queue((task, callback) => { + task(callback) +}, 1) + +const bar = new ProgressBar( + `Generating image thumbnails [:bar] :current/:total :elapsed secs :percent`, + { + total: 0, + width: 30, + } +) + +exports.scheduleJob = async (job, boundActionCreators, pluginOptions) => { + const inputFileKey = job.file.absolutePath.replace(/\./g, `%2E`) + const outputFileKey = job.outputPath.replace(/\./g, `%2E`) + const jobPath = `${inputFileKey}.${outputFileKey}` + + // Check if the job has already been queued. If it has, there's nothing + // to do, return. + if (_.has(toProcess, jobPath)) { + return _.get(toProcess, `${jobPath}.deferred.promise`) + } + + // Check if the output file already exists so we don't redo work. + if (existsSync(job.outputPath)) { + return Promise.resolve(job) + } + + let isQueued = false + if (toProcess[inputFileKey]) { + isQueued = true + } + + // deferred naming comes from https://developer.mozilla.org/en-US/docs/Mozilla/JavaScript_code_modules/Promise.jsm/Deferred + let deferred = {} + deferred.promise = new Promise((resolve, reject) => { + deferred.resolve = resolve + deferred.reject = reject + }) + + totalJobs += 1 + + _.set(toProcess, jobPath, { + job: job, + deferred, + }) + + if (!isQueued) { + q.push(cb => { + runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) + }) + } + + return deferred.promise +} + +function runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) { + const jobs = _.values(toProcess[inputFileKey]) + const findDeferred = job => jobs.find(j => j.job === job).deferred + const { job } = jobs[0] + + // Delete the input key from the toProcess list so more jobs can be queued. + delete toProcess[inputFileKey] + boundActionCreators.createJob( + { + id: `processing image ${job.file.absolutePath}`, + imagesCount: _.values(toProcess[inputFileKey]).length, + }, + { name: `gatsby-plugin-sharp` } + ) + + // We're now processing the file's jobs. + let imagesFinished = 0 + bar.total = totalJobs + + try { + const promises = processFile( + job.file.absolutePath, + jobs.map(job => job.job), + pluginOptions + ).map(promise => + promise + .then(job => { + findDeferred(job).resolve() + }) + .catch(err => { + findDeferred(job).reject({ + err, + message: `Failed to process image ${job.file.absolutePath}`, + }) + }) + .then(() => { + imagesFinished += 1 + bar.tick() + boundActionCreators.setJob( + { + id: `processing image ${job.file.absolutePath}`, + imagesFinished, + }, + { name: `gatsby-plugin-sharp` } + ) + }) + ) + + Promise.all(promises).then(() => { + boundActionCreators.endJob( + { id: `processing image ${job.file.absolutePath}` }, + { name: `gatsby-plugin-sharp` } + ) + cb() + }) + } catch (err) { + jobs.forEach(({ deferred }) => { + deferred.reject({ + err, + message: err.message, + }) + }) + } +} From 6b173655c1b6dea50f2373aa902aaf0c445f1cbc Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 10 Jan 2019 10:57:57 +0100 Subject: [PATCH 03/36] fix eslint --- packages/gatsby-plugin-sharp/src/gatsby-node.js | 7 +++---- packages/gatsby-plugin-sharp/src/index.js | 8 +++++--- packages/gatsby-plugin-sharp/src/scheduler.js | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 559da0a40ffe6..28b913d6304a8 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -14,10 +14,9 @@ exports.onPreInit = ({ actions, cache }, pluginOptions) => { setPluginOptions(pluginOptions) } -exports.onPostBootstrap = async ({ cache }) => { - // JSON.stringify doesn't work on an Map so we need to convert it to an array - return cache.set(`queue`, Array.from(queue)) -} +// JSON.stringify doesn't work on an Map so we need to convert it to an array +exports.onPostBootstrap = async ({ cache }) => + cache.set(`queue`, Array.from(queue)) /** * Execute all unprocessed images on gatsby build diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index 97f295c640c81..a9ee01ca92712 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -194,9 +194,11 @@ function queueImageResizing({ file, args = {}, reporter }) { height, aspectRatio, // finishedPromise is needed to not break our API (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-transformer-sqip/src/extend-node-type.js#L115) - finishedPromise: () => ({ - then: () => scheduleJob(job, boundActionCreators, pluginOptions), - }), + finishedPromise: () => { + return { + then: () => scheduleJob(job, boundActionCreators, pluginOptions), + } + }, originalName: originalName, } } diff --git a/packages/gatsby-plugin-sharp/src/scheduler.js b/packages/gatsby-plugin-sharp/src/scheduler.js index 3c05d21a10ef5..fe3879b9c933d 100644 --- a/packages/gatsby-plugin-sharp/src/scheduler.js +++ b/packages/gatsby-plugin-sharp/src/scheduler.js @@ -1,8 +1,8 @@ -const _ = require("lodash") +const _ = require(`lodash`) const ProgressBar = require(`progress`) const existsSync = require(`fs-exists-cached`).sync const queue = require(`async/queue`) -const processFile = require("./processFile") +const processFile = require(`./processFile`) const toProcess = {} let totalJobs = 0 From 634c8b41309dd68f08110987696f128f3d0af060 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 10 Jan 2019 12:25:48 +0100 Subject: [PATCH 04/36] fix queue caching --- .../gatsby-plugin-sharp/src/gatsby-node.js | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 28b913d6304a8..a49492e9f494f 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -6,49 +6,64 @@ const { } = require(`./index`) const processFile = require(`./processFile`) const { scheduleJob } = require(`./scheduler`) +const fs = require(`fs-extra`) -exports.onPreInit = ({ actions, cache }, pluginOptions) => { - cache.init() +const getQueueFromCache = async cache => { + const rawQueue = await cache.get(`queue`) + + if (!rawQueue) { + return new Map() + } + + return new Map(rawQueue) +} +exports.onPreInit = async ({ actions }, pluginOptions) => { setBoundActionCreators(actions) setPluginOptions(pluginOptions) } // JSON.stringify doesn't work on an Map so we need to convert it to an array -exports.onPostBootstrap = async ({ cache }) => - cache.set(`queue`, Array.from(queue)) +exports.onPostBootstrap = async ({ cache }) => { + const cachedQueue = await getQueueFromCache(cache) + + // merge both queues + for (const [key, job] of cachedQueue) { + queue.set(key, job) + } + + return cache.set(`queue`, Array.from(queue)) +} /** * Execute all unprocessed images on gatsby build */ -exports.onPostBuild = async ( - { cache, boundActionCreators, reporter }, - pluginOptions -) => { - const rawQueue = await cache.get(`queue`) - const queue = new Map(rawQueue) +let promises = [] +exports.onPreBuild = async ({ cache, boundActionCreators }, pluginOptions) => { + const cachedQueue = await getQueueFromCache(cache) let promises = [] - for (const [, job] of queue) { + for (const [, job] of cachedQueue) { promises.push(scheduleJob(job, boundActionCreators, pluginOptions)) } - - return Promise.all(promises).catch(({ err, message }) => { - reportError(message || err.message, err, reporter) - }) } +/** + * wait for all images to be processed + */ +exports.onPostBuild = ({ cache, reporter }) => + Promise.all(promises) + .then(() => cache.set(`queue`, [])) + .catch(({ err, message }) => { + reportError(message || err.message, err, reporter) + }) + /** * Build images on the fly when they are requested by the browser */ exports.onCreateDevServer = ({ app, cache }, pluginOptions) => { app.use(async (req, res, next) => { - const rawQueue = await cache.get(`queue`) - if (!rawQueue) { - return next() - } - - const queue = new Map(rawQueue) + const queue = await getQueueFromCache(cache) if (!queue.has(req.originalUrl)) { return next() } From 579a04054bc2aaa1ffea5f46b9f807221dff0aed Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 10 Jan 2019 12:26:15 +0100 Subject: [PATCH 05/36] fix queue caching --- packages/gatsby-plugin-sharp/src/gatsby-node.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index a49492e9f494f..a7bc135a1fd25 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -6,7 +6,6 @@ const { } = require(`./index`) const processFile = require(`./processFile`) const { scheduleJob } = require(`./scheduler`) -const fs = require(`fs-extra`) const getQueueFromCache = async cache => { const rawQueue = await cache.get(`queue`) From 9c30bbef820d6bfc7f5479a0d08904c9d2273777 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 10 Jan 2019 13:40:58 +0100 Subject: [PATCH 06/36] move comments --- packages/gatsby-plugin-sharp/src/gatsby-node.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index a7bc135a1fd25..67e73303d9351 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -22,7 +22,9 @@ exports.onPreInit = async ({ actions }, pluginOptions) => { setPluginOptions(pluginOptions) } -// JSON.stringify doesn't work on an Map so we need to convert it to an array +/** + * save queue to the cache + */ exports.onPostBootstrap = async ({ cache }) => { const cachedQueue = await getQueueFromCache(cache) @@ -31,6 +33,7 @@ exports.onPostBootstrap = async ({ cache }) => { queue.set(key, job) } + // JSON.stringify doesn't work on an Map so we need to convert it to an array return cache.set(`queue`, Array.from(queue)) } From 9e73a6a725f21ab31304b73bf096472d93c7332a Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Fri, 11 Jan 2019 07:38:59 +0100 Subject: [PATCH 07/36] fix comments --- packages/gatsby-plugin-sharp/.gitignore | 7 +------ packages/gatsby-plugin-sharp/src/gatsby-node.js | 5 ++++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/gatsby-plugin-sharp/.gitignore b/packages/gatsby-plugin-sharp/.gitignore index a331b6826346d..f8e91069da686 100644 --- a/packages/gatsby-plugin-sharp/.gitignore +++ b/packages/gatsby-plugin-sharp/.gitignore @@ -1,7 +1,2 @@ -/index.js /node_modules -/gatsby-node.js -/duotone.js -/queue.js -/processFile.js -/scheduler.js +/*.js diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 67e73303d9351..6dfd89373e01a 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -44,7 +44,6 @@ let promises = [] exports.onPreBuild = async ({ cache, boundActionCreators }, pluginOptions) => { const cachedQueue = await getQueueFromCache(cache) - let promises = [] for (const [, job] of cachedQueue) { promises.push(scheduleJob(job, boundActionCreators, pluginOptions)) } @@ -74,6 +73,10 @@ exports.onCreateDevServer = ({ app, cache }, pluginOptions) => { // wait until the file has been processed and saved to disk await Promise.all(processFile(job.file.absolutePath, [job], pluginOptions)) + queue.delete(req.originalUrl) + + // JSON.stringify doesn't work on an Map so we need to convert it to an array + cache.set(`queue`, Array.from(queue)) return res.sendFile(job.outputPath) }) From b7fbb53577a12ce820c1f1705a5823b8068c3b0a Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 24 Jan 2019 21:55:42 +0100 Subject: [PATCH 08/36] update cache on query change --- .../gatsby-plugin-sharp/src/gatsby-node.js | 26 ++++++++++++------- packages/gatsby/src/commands/develop.js | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 6dfd89373e01a..e9e3df1759782 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -17,15 +17,7 @@ const getQueueFromCache = async cache => { return new Map(rawQueue) } -exports.onPreInit = async ({ actions }, pluginOptions) => { - setBoundActionCreators(actions) - setPluginOptions(pluginOptions) -} - -/** - * save queue to the cache - */ -exports.onPostBootstrap = async ({ cache }) => { +const saveQueueToCache = async cache => { const cachedQueue = await getQueueFromCache(cache) // merge both queues @@ -37,6 +29,16 @@ exports.onPostBootstrap = async ({ cache }) => { return cache.set(`queue`, Array.from(queue)) } +exports.onPreInit = async ({ actions }, pluginOptions) => { + setBoundActionCreators(actions) + setPluginOptions(pluginOptions) +} + +/** + * save queue to the cache + */ +exports.onPostBootstrap = async ({ cache, store }) => saveQueueToCache(cache) + /** * Execute all unprocessed images on gatsby build */ @@ -62,7 +64,11 @@ exports.onPostBuild = ({ cache, reporter }) => /** * Build images on the fly when they are requested by the browser */ -exports.onCreateDevServer = ({ app, cache }, pluginOptions) => { +exports.onCreateDevServer = async ({ app, cache, compiler }, pluginOptions) => { + compiler.hooks.done.tap(`Gatsby`, () => { + saveQueueToCache(cache) + }) + app.use(async (req, res, next) => { const queue = await getQueueFromCache(cache) if (!queue.has(req.originalUrl)) { diff --git a/packages/gatsby/src/commands/develop.js b/packages/gatsby/src/commands/develop.js index c88dc577428d1..5250b1adf5f1f 100644 --- a/packages/gatsby/src/commands/develop.js +++ b/packages/gatsby/src/commands/develop.js @@ -182,7 +182,7 @@ async function startServer(program) { }) } - await apiRunnerNode(`onCreateDevServer`, { app }) + await apiRunnerNode(`onCreateDevServer`, { app, compiler }) // Render an HTML page and serve it. app.use((req, res, next) => { From b243d0f8c11d2875686edd70762c8c5b05544a8a Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 24 Jan 2019 21:58:05 +0100 Subject: [PATCH 09/36] update webpack tap name --- packages/gatsby-plugin-sharp/src/gatsby-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index e9e3df1759782..a9a30b0815e9c 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -65,7 +65,7 @@ exports.onPostBuild = ({ cache, reporter }) => * Build images on the fly when they are requested by the browser */ exports.onCreateDevServer = async ({ app, cache, compiler }, pluginOptions) => { - compiler.hooks.done.tap(`Gatsby`, () => { + compiler.hooks.done.tap(`gatsby-plugin-sharp`, () => { saveQueueToCache(cache) }) From 286928161eae5e8a906f8a10d9dade5048fc7d7d Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Fri, 25 Jan 2019 16:32:24 +0100 Subject: [PATCH 10/36] review changes --- .../gatsby-plugin-sharp/src/gatsby-node.js | 21 ++++++++++--------- packages/gatsby/src/commands/develop.js | 2 +- www/README.md | 12 ----------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index a9a30b0815e9c..ff9e92df05f15 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -1,7 +1,7 @@ const { setBoundActionCreators, setPluginOptions, - queue, + queue: jobQueue, reportError, } = require(`./index`) const processFile = require(`./processFile`) @@ -17,12 +17,14 @@ const getQueueFromCache = async cache => { return new Map(rawQueue) } -const saveQueueToCache = async cache => { +const saveQueueToCache = async (cache, queue) => { const cachedQueue = await getQueueFromCache(cache) // merge both queues for (const [key, job] of cachedQueue) { - queue.set(key, job) + if (!queue.has(key)) { + queue.set(key, job) + } } // JSON.stringify doesn't work on an Map so we need to convert it to an array @@ -37,7 +39,8 @@ exports.onPreInit = async ({ actions }, pluginOptions) => { /** * save queue to the cache */ -exports.onPostBootstrap = async ({ cache, store }) => saveQueueToCache(cache) +exports.onPostBootstrap = async ({ cache, store }) => + saveQueueToCache(cache, jobQueue) /** * Execute all unprocessed images on gatsby build @@ -64,10 +67,8 @@ exports.onPostBuild = ({ cache, reporter }) => /** * Build images on the fly when they are requested by the browser */ -exports.onCreateDevServer = async ({ app, cache, compiler }, pluginOptions) => { - compiler.hooks.done.tap(`gatsby-plugin-sharp`, () => { - saveQueueToCache(cache) - }) +exports.onCreateDevServer = async ({ app, cache, emitter }, pluginOptions) => { + emitter(`QUERY_QUEUE_DRAINED`, () => saveQueueToCache(cache, jobQueue)) app.use(async (req, res, next) => { const queue = await getQueueFromCache(cache) @@ -79,10 +80,10 @@ exports.onCreateDevServer = async ({ app, cache, compiler }, pluginOptions) => { // wait until the file has been processed and saved to disk await Promise.all(processFile(job.file.absolutePath, [job], pluginOptions)) + // remove job from queue because it has been processed queue.delete(req.originalUrl) - // JSON.stringify doesn't work on an Map so we need to convert it to an array - cache.set(`queue`, Array.from(queue)) + await saveQueueToCache(cache, queue) return res.sendFile(job.outputPath) }) diff --git a/packages/gatsby/src/commands/develop.js b/packages/gatsby/src/commands/develop.js index 5250b1adf5f1f..c88dc577428d1 100644 --- a/packages/gatsby/src/commands/develop.js +++ b/packages/gatsby/src/commands/develop.js @@ -182,7 +182,7 @@ async function startServer(program) { }) } - await apiRunnerNode(`onCreateDevServer`, { app, compiler }) + await apiRunnerNode(`onCreateDevServer`, { app }) // Render an HTML page and serve it. app.use((req, res, next) => { diff --git a/www/README.md b/www/README.md index 52e76d76431ab..8e44fff478f7d 100644 --- a/www/README.md +++ b/www/README.md @@ -39,15 +39,3 @@ GATSBY_FEEDBACK_KEY_STARTERLIB=ADD_KEY ``` If there's a problem with the feedback widgets, please open an issue in the repo. - -## Running slow build? (Screenshots placeholder) - -If you are not working on starter or site showcase, it might be beneficial to use a placeholder image instead of actual screenshots. It will skip downloading screenshots and generating responsive images for all screenshots and replace them with a placeholder image. - -Add the following env variable to your `.env.development` file to enable placeholder behaviour: - -``` -GATSBY_SCREENSHOT_PLACEHOLDER=true -``` - -For more information checkout [`gatsby-transformer-screenshot` docs](http://www.gatsbyjs.org/packages/gatsby-transformer-screenshot#placeholder-image). From b415b6730fc2c08cada553809e38d32f0815dd6c Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Fri, 25 Jan 2019 17:34:41 +0100 Subject: [PATCH 11/36] review changes --- packages/gatsby-plugin-sharp/src/gatsby-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index ff9e92df05f15..3bea0fbc1caef 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -68,7 +68,7 @@ exports.onPostBuild = ({ cache, reporter }) => * Build images on the fly when they are requested by the browser */ exports.onCreateDevServer = async ({ app, cache, emitter }, pluginOptions) => { - emitter(`QUERY_QUEUE_DRAINED`, () => saveQueueToCache(cache, jobQueue)) + emitter.on(`QUERY_QUEUE_DRAINED`, () => saveQueueToCache(cache, jobQueue)) app.use(async (req, res, next) => { const queue = await getQueueFromCache(cache) From 981a02dd1bd2e4a086c9c2d8305802ce85742231 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Fri, 1 Feb 2019 15:27:09 +0100 Subject: [PATCH 12/36] fix review --- .../gatsby-plugin-sharp/src/gatsby-node.js | 2 +- .../gatsby-plugin-sharp/src/processFile.js | 19 ++++--------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 3bea0fbc1caef..0a674e21e506f 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -31,7 +31,7 @@ const saveQueueToCache = async (cache, queue) => { return cache.set(`queue`, Array.from(queue)) } -exports.onPreInit = async ({ actions }, pluginOptions) => { +exports.onPreBootstrap = async ({ actions }, pluginOptions) => { setBoundActionCreators(actions) setPluginOptions(pluginOptions) } diff --git a/packages/gatsby-plugin-sharp/src/processFile.js b/packages/gatsby-plugin-sharp/src/processFile.js index f4ee6c7449b62..a08ba843ccf78 100644 --- a/packages/gatsby-plugin-sharp/src/processFile.js +++ b/packages/gatsby-plugin-sharp/src/processFile.js @@ -7,17 +7,6 @@ const imageminMozjpeg = require(`imagemin-mozjpeg`) const imageminPngquant = require(`imagemin-pngquant`) const imageminWebp = require(`imagemin-webp`) -const writeFileAsync = (file, buffer) => - new Promise((resovle, reject) => { - fs.writeFile(file, buffer, err => { - if (err) { - return reject(err) - } - - return resovle() - }) - }) - // Try to enable the use of SIMD instructions. Seems to provide a smallish // speedup on resizing heavy loads (~10%). Sharp disables this feature by // default as there's been problems with segfaulting in the past but we'll be @@ -126,7 +115,7 @@ module.exports = (file, transforms, options = {}) => { } if ( - (file.extension === `webp` && args.toFormat === ``) || + (transformFile.extension === `webp` && args.toFormat === ``) || args.toFormat === `webp` ) { await compressWebP(clonedPipeline, outputPath, args) @@ -155,7 +144,7 @@ const compressPng = (pipeline, outputPath, options) => }), ], }) - .then(imageminBuffer => writeFileAsync(outputPath, imageminBuffer)) + .then(imageminBuffer => fs.writeFile(outputPath, imageminBuffer)) ) const compressJpg = (pipeline, outputPath, options) => @@ -169,7 +158,7 @@ const compressJpg = (pipeline, outputPath, options) => }), ], }) - .then(imageminBuffer => writeFileAsync(outputPath, imageminBuffer)) + .then(imageminBuffer => fs.writeFile(outputPath, imageminBuffer)) ) const compressWebP = (pipeline, outputPath, options) => @@ -178,5 +167,5 @@ const compressWebP = (pipeline, outputPath, options) => .buffer(sharpBuffer, { plugins: [imageminWebp({ quality: options.quality })], }) - .then(imageminBuffer => writeFileAsync(outputPath, imageminBuffer)) + .then(imageminBuffer => fs.writeFile(outputPath, imageminBuffer)) ) From e01844d97456e9a84001ebde55f03206f70cb2e6 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Mon, 11 Feb 2019 05:18:23 -0800 Subject: [PATCH 13/36] make cache smaller & refator toFormat --- packages/gatsby-plugin-sharp/src/index.js | 31 ++++++++++++------- .../gatsby-plugin-sharp/src/processFile.js | 21 +++---------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index a9ee01ca92712..c7098e54775f1 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -75,13 +75,22 @@ const generalArgs = { sizeByPixelDensity: false, } -const healOptions = (args, defaultArgs) => { +const healOptions = (args, fileExtension, defaultArgs = {}) => { let options = _.defaults({}, args, defaultArgs, generalArgs) options.quality = parseInt(options.quality, 10) options.pngCompressionLevel = parseInt(options.pngCompressionLevel, 10) options.pngCompressionSpeed = parseInt(options.pngCompressionSpeed, 10) options.toFormat = options.toFormat.toLowerCase() + // when toFormat is not set we set it based on fileExtension + if (options.toFormat === ``) { + options.toFormat = fileExtension.toLowerCase() + + if (fileExtension === `jpeg`) { + options.toFormat = `jpg` + } + } + // only set width to 400 if neither width nor height is passed if (options.width === undefined && options.height === undefined) { options.width = 400 @@ -104,7 +113,7 @@ const healOptions = (args, defaultArgs) => { } function queueImageResizing({ file, args = {}, reporter }) { - const options = healOptions(args, {}) + const options = healOptions(args, file.extension) // Filter out false args, and args not for this extension and put width at // end (for the file path) const pairedArgs = _.toPairs(args) @@ -178,7 +187,9 @@ function queueImageResizing({ file, args = {}, reporter }) { // Create job and add it to the queue, the queue will be processed inside gatsby-node.js const job = { - file, + file: { + absolutePath: file.absolutePath, + }, args: options, inputPath: file.absolutePath, outputPath: filePath, @@ -204,7 +215,7 @@ function queueImageResizing({ file, args = {}, reporter }) { } async function generateBase64({ file, args, reporter }) { - const options = healOptions(args, { width: 20 }) + const options = healOptions(args, file.extension, { width: 20 }) let pipeline try { pipeline = sharp(file.absolutePath).rotate() @@ -240,11 +251,7 @@ async function generateBase64({ file, args, reporter }) { // duotone if (options.duotone) { - pipeline = await duotone( - options.duotone, - args.toFormat || file.extension, - pipeline - ) + pipeline = await duotone(options.duotone, args.toFormat, pipeline) } const { data: buffer, info } = await pipeline.toBuffer({ resolveWithObject: true, @@ -288,7 +295,7 @@ async function base64(arg) { } async function fluid({ file, args = {}, reporter, cache }) { - const options = healOptions(args, {}) + const options = healOptions(args, file.extension) // Account for images with a high pixel density. We assume that these types of // images are intended to be displayed at their native resolution. let metadata @@ -463,7 +470,7 @@ async function fluid({ file, args = {}, reporter, cache }) { } async function fixed({ file, args = {}, reporter, cache }) { - const options = healOptions(args, {}) + const options = healOptions(args, file.extension) // if no width is passed, we need to resize the image based on the passed height const fixedDimension = options.width === undefined ? `height` : `width` @@ -573,7 +580,7 @@ async function notMemoizedtraceSVG({ file, args, fileArgs, reporter }) { turnPolicy: potrace.Potrace.TURNPOLICY_MAJORITY, } const optionsSVG = _.defaults(args, defaultArgs) - const options = healOptions(fileArgs, {}) + const options = healOptions(fileArgs, file.extension) let pipeline try { pipeline = sharp(file.absolutePath).rotate() diff --git a/packages/gatsby-plugin-sharp/src/processFile.js b/packages/gatsby-plugin-sharp/src/processFile.js index a08ba843ccf78..cc31252bc0840 100644 --- a/packages/gatsby-plugin-sharp/src/processFile.js +++ b/packages/gatsby-plugin-sharp/src/processFile.js @@ -29,7 +29,7 @@ module.exports = (file, transforms, options = {}) => { } return transforms.map(async transform => { - const { file: transformFile, outputPath, args } = transform + const { outputPath, args } = transform debug(`Start processing ${outputPath}`) let clonedPipeline = transforms.length > 1 ? pipeline.clone() : pipeline @@ -87,16 +87,13 @@ module.exports = (file, transforms, options = {}) => { if (args.duotone) { clonedPipeline = await duotone( args.duotone, - args.toFormat || transformFile.extension, + args.toFormat, clonedPipeline ) } // lets decide how we want to save this transform - if ( - (transformFile.extension === `png` && args.toFormat === ``) || - args.toFormat === `png` - ) { + if (args.toFormat === `png`) { await compressPng(clonedPipeline, outputPath, { ...args, stripMetadata: options.stripMetadata, @@ -104,20 +101,12 @@ module.exports = (file, transforms, options = {}) => { return transform } - if ( - options.useMozJpeg && - ((transformFile.extension === `jpg` && args.toFormat === ``) || - (transformFile.extension === `jpeg` && args.toFormat === ``) || - args.toFormat === `jpg`) - ) { + if (options.useMozJpeg && args.toFormat === `jpg`) { await compressJpg(clonedPipeline, outputPath, args) return transform } - if ( - (transformFile.extension === `webp` && args.toFormat === ``) || - args.toFormat === `webp` - ) { + if (args.toFormat === `webp`) { await compressWebP(clonedPipeline, outputPath, args) return transform } From bd5e8018e8a2331f25b171b90beaafe309e9c4b7 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Mon, 11 Feb 2019 06:39:10 -0800 Subject: [PATCH 14/36] fix testsf --- .../__tests__/lazy-image-build/develop.js | 11 ++++++++--- packages/gatsby-plugin-sharp/src/gatsby-node.js | 2 +- packages/gatsby-plugin-sharp/src/index.js | 8 +++++++- .../src/{processFile.js => process-file.js} | 0 packages/gatsby-plugin-sharp/src/scheduler.js | 2 +- 5 files changed, 17 insertions(+), 6 deletions(-) rename packages/gatsby-plugin-sharp/src/{processFile.js => process-file.js} (100%) diff --git a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js index f0f4f8eea47b6..506fa62f87f19 100644 --- a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js +++ b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js @@ -24,7 +24,8 @@ describe(`Lazy images`, () => { const { kill } = await createDevServer() const response = await request( - `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a2541/gatsby-astronaut.png`, + `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a484e/gatsby-astronaut.png`, + // `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a2541/gatsby-astronaut.png`, { resolveWithFullResponse: true, } @@ -33,18 +34,22 @@ describe(`Lazy images`, () => { await kill() expect(response.statusCode).toBe(200) + expect(response.headers[`content-type`]).toBe(`image/png`) const images = glob.sync(`${basePath}/public/**/*.png`) - expect(images.length).toBe(6) + expect(images.length).toBe(1) }) test(`should process the rest of images on build`, async () => { + let images = glob.sync(`${basePath}/public/**/*.png`) + expect(images.length).toBe(1) + await execa(`yarn`, [`build`], { cwd: basePath, env: { NODE_ENV: `production` }, }) - const images = glob.sync(`${basePath}/public/**/*.png`) + images = glob.sync(`${basePath}/public/**/*.png`) expect(images.length).toBe(6) }) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 0a674e21e506f..34e9d898cbee2 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -4,7 +4,7 @@ const { queue: jobQueue, reportError, } = require(`./index`) -const processFile = require(`./processFile`) +const processFile = require(`./process-file`) const { scheduleJob } = require(`./scheduler`) const getQueueFromCache = async cache => { diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index c7098e54775f1..8a306e4949327 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -207,7 +207,13 @@ function queueImageResizing({ file, args = {}, reporter }) { // finishedPromise is needed to not break our API (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-transformer-sqip/src/extend-node-type.js#L115) finishedPromise: () => { return { - then: () => scheduleJob(job, boundActionCreators, pluginOptions), + then: () => + scheduleJob(job, boundActionCreators, pluginOptions).then( + (...res) => { + queue.delete(prefixedSrc) + return res + } + ), } }, originalName: originalName, diff --git a/packages/gatsby-plugin-sharp/src/processFile.js b/packages/gatsby-plugin-sharp/src/process-file.js similarity index 100% rename from packages/gatsby-plugin-sharp/src/processFile.js rename to packages/gatsby-plugin-sharp/src/process-file.js diff --git a/packages/gatsby-plugin-sharp/src/scheduler.js b/packages/gatsby-plugin-sharp/src/scheduler.js index fe3879b9c933d..9930c2d893f62 100644 --- a/packages/gatsby-plugin-sharp/src/scheduler.js +++ b/packages/gatsby-plugin-sharp/src/scheduler.js @@ -2,7 +2,7 @@ const _ = require(`lodash`) const ProgressBar = require(`progress`) const existsSync = require(`fs-exists-cached`).sync const queue = require(`async/queue`) -const processFile = require(`./processFile`) +const processFile = require(`./process-file`) const toProcess = {} let totalJobs = 0 From fbe37e9f9343f1b611779fe2faf10f13cb747c0d Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Mon, 11 Feb 2019 11:20:08 -0800 Subject: [PATCH 15/36] fix schedulejob on demand --- packages/gatsby-plugin-sharp/src/gatsby-node.js | 2 +- packages/gatsby-plugin-sharp/src/index.js | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 34e9d898cbee2..f032962078a91 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -31,7 +31,7 @@ const saveQueueToCache = async (cache, queue) => { return cache.set(`queue`, Array.from(queue)) } -exports.onPreBootstrap = async ({ actions }, pluginOptions) => { +exports.onPreBootstrap = ({ actions }, pluginOptions) => { setBoundActionCreators(actions) setPluginOptions(pluginOptions) } diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index 8a306e4949327..cc610c7b0e7b2 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -205,16 +205,13 @@ function queueImageResizing({ file, args = {}, reporter }) { height, aspectRatio, // finishedPromise is needed to not break our API (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-transformer-sqip/src/extend-node-type.js#L115) - finishedPromise: () => { - return { - then: () => - scheduleJob(job, boundActionCreators, pluginOptions).then( - (...res) => { - queue.delete(prefixedSrc) - return res - } - ), - } + finishedPromise: { + then: (resolve, reject) => { + scheduleJob(job, boundActionCreators, pluginOptions).then(() => { + queue.delete(prefixedSrc) + resolve() + }, reject) + }, }, originalName: originalName, } From 8e1d7145ab0658f0da286d7995c4e7b4dcdcf834 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Mon, 11 Feb 2019 23:49:37 -0800 Subject: [PATCH 16/36] update snapshots --- .../__tests__/lazy-image-build/develop.js | 1 - .../src/__tests__/__snapshots__/index.js.snap | 22 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js index 506fa62f87f19..356dd0fbcf594 100644 --- a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js +++ b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js @@ -25,7 +25,6 @@ describe(`Lazy images`, () => { const response = await request( `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a484e/gatsby-astronaut.png`, - // `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a2541/gatsby-astronaut.png`, { resolveWithFullResponse: true, } diff --git a/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap b/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap index d6a1fc5a2137c..dcad8bd999447 100644 --- a/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap +++ b/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap @@ -15,14 +15,14 @@ Object { "aspectRatio": 2.0661764705882355, "base64": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAABYlAAAWJQFJUiTwAAABAklEQVQoz61R7WrCQBDM+7+Cv1qhlIogoUWMoH8KrfgOLVXRKDV4d7kkl6/pbsxpqBIo9mBuZ7N3k51bB7TKsrwZVsexyS2rKerwFhYFVI1DnkMQOHIuCaLB+ayuwXlMaApXgvMoxjTU8MIQE4rMx8SnNffUkU/qM/bbiPAeRfgw5n8tB+QgO1kmUvwROdm0kYHmUNoe+NoU2wZTdVjQH7Isg9YRpFIXor8vpGmKjb/FQQgslit8fi3wvQ/OHXLhxRvDHY7Qd5+x3e2udmNzTUN4fZvhoddH566L+8cn9AZuVcvpLR3e9kEAISWEkDAmbRXkGCcJ1r4PRRNPEkPu9Kn2Azs0CqJYU3JKAAAAAElFTkSuQmCC", "density": 144, - "originalImg": "/static/1234/a3030/test.png", + "originalImg": "/static/1234/29cdc/test.png", "originalName": undefined, "presentationHeight": 68, "presentationWidth": 141, "sizes": "(max-width: 141px) 100vw, 141px", - "src": "/static/1234/a3030/test.png", - "srcSet": "/static/1234/56b42/test.png 200w, -/static/1234/a3030/test.png 281w", + "src": "/static/1234/29cdc/test.png", + "srcSet": "/static/1234/223da/test.png 200w, +/static/1234/29cdc/test.png 281w", "srcSetType": "image/png", } `; @@ -32,14 +32,14 @@ Object { "aspectRatio": 2.0661764705882355, "base64": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAABYlAAAWJQFJUiTwAAABAklEQVQoz61R7WrCQBDM+7+Cv1qhlIogoUWMoH8KrfgOLVXRKDV4d7kkl6/pbsxpqBIo9mBuZ7N3k51bB7TKsrwZVsexyS2rKerwFhYFVI1DnkMQOHIuCaLB+ayuwXlMaApXgvMoxjTU8MIQE4rMx8SnNffUkU/qM/bbiPAeRfgw5n8tB+QgO1kmUvwROdm0kYHmUNoe+NoU2wZTdVjQH7Isg9YRpFIXor8vpGmKjb/FQQgslit8fi3wvQ/OHXLhxRvDHY7Qd5+x3e2udmNzTUN4fZvhoddH566L+8cn9AZuVcvpLR3e9kEAISWEkDAmbRXkGCcJ1r4PRRNPEkPu9Kn2Azs0CqJYU3JKAAAAAElFTkSuQmCC", "density": 144, - "originalImg": "/static/1234/67b29/test.png", + "originalImg": "/static/1234/55fae/test.png", "originalName": undefined, "presentationHeight": 136, "presentationWidth": 281, "sizes": "(max-width: 281px) 100vw, 281px", - "src": "/static/1234/67b29/test.png", - "srcSet": "/static/1234/4df82/test.png 200w, -/static/1234/67b29/test.png 281w", + "src": "/static/1234/55fae/test.png", + "srcSet": "/static/1234/6d94b/test.png 200w, +/static/1234/55fae/test.png 281w", "srcSetType": "image/png", } `; @@ -49,13 +49,13 @@ Object { "aspectRatio": 1, "base64": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAIAAAAC64paAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAGElEQVQ4y2P4TwFgGNU8qnlU86jmgdUMAOBEq42KgAyMAAAAAElFTkSuQmCC", "density": 72, - "originalImg": "/static/1234/4ce1a/test.png", + "originalImg": "/static/1234/7bd23/test.png", "originalName": undefined, "presentationHeight": 1, "presentationWidth": 1, "sizes": "(max-width: 1px) 100vw, 1px", - "src": "/static/1234/4ce1a/test.png", - "srcSet": "/static/1234/4ce1a/test.png 1w", + "src": "/static/1234/7bd23/test.png", + "srcSet": "/static/1234/7bd23/test.png 1w", "srcSetType": "image/png", } `; From c6509aa90979b9325a67d398aa939355b4bf8413 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Feb 2019 00:19:00 -0800 Subject: [PATCH 17/36] add tests to opt out of lazy image resolving --- .../src/__tests__/__snapshots__/index.js.snap | 63 +++++++++++++++++++ .../src/__tests__/index.js | 16 ++++- .../gatsby-plugin-sharp/src/gatsby-node.js | 2 +- packages/gatsby-plugin-sharp/src/index.js | 3 - packages/gatsby-plugin-sharp/src/scheduler.js | 12 ++-- 5 files changed, 85 insertions(+), 11 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap b/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap index dcad8bd999447..7d0833ef8be69 100644 --- a/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap +++ b/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap @@ -70,3 +70,66 @@ Object { "width": 746, } `; + +exports[`gatsby-plugin-sharp queueImageResizing should process immediately when asked 1`] = ` +[MockFunction] { + "calls": Array [ + Array [ + Object { + "args": Object { + "base64": false, + "duotone": false, + "grayscale": false, + "jpegProgressive": true, + "maxWidth": 800, + "pathPrefix": "", + "pngCompressionLevel": 9, + "pngCompressionSpeed": 4, + "quality": 50, + "sizeByPixelDensity": false, + "toFormat": "png", + "width": 3, + }, + "inputPath": "/home/ward/projects/os/gatsby/packages/gatsby-plugin-sharp/src/__tests__/images/144-density.png", + "outputPath": "/home/ward/projects/os/gatsby/public/static/1234/39ca0/test.png", + "src": "/static/1234/39ca0/test.png", + }, + Object { + "addThirdPartySchema": [Function], + "createJob": [Function], + "createNode": [Function], + "createNodeField": [Function], + "createPage": [Function], + "createPageDependency": [Function], + "createParentChildLink": [Function], + "createRedirect": [Function], + "deleteComponentsDependencies": [Function], + "deleteNode": [Function], + "deleteNodes": [Function], + "deletePage": [Function], + "endJob": [Function], + "replaceComponentQuery": [Function], + "replaceStaticQuery": [Function], + "replaceWebpackConfig": [Function], + "setBabelOptions": [Function], + "setBabelPlugin": [Function], + "setBabelPreset": [Function], + "setJob": [Function], + "setPluginStatus": [Function], + "setWebpackConfig": [Function], + "touchNode": [Function], + }, + Object { + "stripMetadata": true, + "useMozJpeg": false, + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], +} +`; diff --git a/packages/gatsby-plugin-sharp/src/__tests__/index.js b/packages/gatsby-plugin-sharp/src/__tests__/index.js index a1d38df161a7f..863355fc46cd8 100644 --- a/packages/gatsby-plugin-sharp/src/__tests__/index.js +++ b/packages/gatsby-plugin-sharp/src/__tests__/index.js @@ -1,5 +1,6 @@ const path = require(`path`) const fs = require(`fs-extra`) +jest.mock(`../scheduler`) jest.mock(`async/queue`, () => () => { return { @@ -16,6 +17,7 @@ const { queueImageResizing, getImageSize, } = require(`../`) +const { scheduleJob } = require(`../scheduler`) describe(`gatsby-plugin-sharp`, () => { const args = { @@ -50,7 +52,7 @@ describe(`gatsby-plugin-sharp`, () => { // test name encoding with various characters const testName = `spaces and '"@#$%^&,` - const queueResult = await queueImageResizing({ + const queueResult = queueImageResizing({ file: getFileObject( path.join(__dirname, `images/144-density.png`), testName @@ -68,6 +70,18 @@ describe(`gatsby-plugin-sharp`, () => { expect(testName.match(/[!@#$^&," ]/)).not.toBe(false) expect(queueResultName.match(/[!@#$^&," ]/)).not.toBe(true) }) + + it(`should process immediately when asked`, async () => { + scheduleJob.mockResolvedValue(Promise.resolve()) + const result = queueImageResizing({ + file: getFileObject(path.join(__dirname, `images/144-density.png`)), + args: { width: 3 }, + }) + + await result.finishedPromise + + expect(scheduleJob).toMatchSnapshot() + }) }) describe(`fluid`, () => { diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index f032962078a91..afa68ce268a5a 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -79,7 +79,7 @@ exports.onCreateDevServer = async ({ app, cache, emitter }, pluginOptions) => { const job = queue.get(req.originalUrl) // wait until the file has been processed and saved to disk - await Promise.all(processFile(job.file.absolutePath, [job], pluginOptions)) + await Promise.all(processFile(job.inputPath, [job], pluginOptions)) // remove job from queue because it has been processed queue.delete(req.originalUrl) diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index cc610c7b0e7b2..eb3fdb513c58c 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -187,9 +187,6 @@ function queueImageResizing({ file, args = {}, reporter }) { // Create job and add it to the queue, the queue will be processed inside gatsby-node.js const job = { - file: { - absolutePath: file.absolutePath, - }, args: options, inputPath: file.absolutePath, outputPath: filePath, diff --git a/packages/gatsby-plugin-sharp/src/scheduler.js b/packages/gatsby-plugin-sharp/src/scheduler.js index 9930c2d893f62..16697bdc6ed9f 100644 --- a/packages/gatsby-plugin-sharp/src/scheduler.js +++ b/packages/gatsby-plugin-sharp/src/scheduler.js @@ -19,7 +19,7 @@ const bar = new ProgressBar( ) exports.scheduleJob = async (job, boundActionCreators, pluginOptions) => { - const inputFileKey = job.file.absolutePath.replace(/\./g, `%2E`) + const inputFileKey = job.inputPath.replace(/\./g, `%2E`) const outputFileKey = job.outputPath.replace(/\./g, `%2E`) const jobPath = `${inputFileKey}.${outputFileKey}` @@ -71,7 +71,7 @@ function runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) { delete toProcess[inputFileKey] boundActionCreators.createJob( { - id: `processing image ${job.file.absolutePath}`, + id: `processing image ${job.inputPath}`, imagesCount: _.values(toProcess[inputFileKey]).length, }, { name: `gatsby-plugin-sharp` } @@ -83,7 +83,7 @@ function runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) { try { const promises = processFile( - job.file.absolutePath, + job.inputPath, jobs.map(job => job.job), pluginOptions ).map(promise => @@ -94,7 +94,7 @@ function runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) { .catch(err => { findDeferred(job).reject({ err, - message: `Failed to process image ${job.file.absolutePath}`, + message: `Failed to process image ${job.inputPath}`, }) }) .then(() => { @@ -102,7 +102,7 @@ function runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) { bar.tick() boundActionCreators.setJob( { - id: `processing image ${job.file.absolutePath}`, + id: `processing image ${job.inputPath}`, imagesFinished, }, { name: `gatsby-plugin-sharp` } @@ -112,7 +112,7 @@ function runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) { Promise.all(promises).then(() => { boundActionCreators.endJob( - { id: `processing image ${job.file.absolutePath}` }, + { id: `processing image ${job.inputPath}` }, { name: `gatsby-plugin-sharp` } ) cb() From 19329355a55c1eca3d6982a571ffe8032c3cecd1 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Feb 2019 00:23:02 -0800 Subject: [PATCH 18/36] Use schedulejob instead of processFile in devmode --- packages/gatsby-plugin-sharp/src/gatsby-node.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index afa68ce268a5a..ce99d05c0ef5a 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -67,7 +67,10 @@ exports.onPostBuild = ({ cache, reporter }) => /** * Build images on the fly when they are requested by the browser */ -exports.onCreateDevServer = async ({ app, cache, emitter }, pluginOptions) => { +exports.onCreateDevServer = async ( + { app, cache, emitter, boundActionCreators }, + pluginOptions +) => { emitter.on(`QUERY_QUEUE_DRAINED`, () => saveQueueToCache(cache, jobQueue)) app.use(async (req, res, next) => { @@ -79,7 +82,7 @@ exports.onCreateDevServer = async ({ app, cache, emitter }, pluginOptions) => { const job = queue.get(req.originalUrl) // wait until the file has been processed and saved to disk - await Promise.all(processFile(job.inputPath, [job], pluginOptions)) + await scheduleJob(job, boundActionCreators, pluginOptions) // remove job from queue because it has been processed queue.delete(req.originalUrl) From 4627ea20eaf546d84e7bb24cf6a901df023550ca Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Feb 2019 00:23:18 -0800 Subject: [PATCH 19/36] Use schedulejob instead of processFile in devmode Speed up multiple requests with some input files --- packages/gatsby-plugin-sharp/src/gatsby-node.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index ce99d05c0ef5a..7d164a63d0ee9 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -4,7 +4,6 @@ const { queue: jobQueue, reportError, } = require(`./index`) -const processFile = require(`./process-file`) const { scheduleJob } = require(`./scheduler`) const getQueueFromCache = async cache => { From 7d23e635a169affbaea237bb7ae7897c3afe3e5a Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Feb 2019 01:32:23 -0800 Subject: [PATCH 20/36] hide status on develop (clutters the cli) --- .../gatsby-plugin-sharp/src/gatsby-node.js | 2 +- packages/gatsby-plugin-sharp/src/scheduler.js | 30 ++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 7d164a63d0ee9..1979d060b8bcf 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -81,7 +81,7 @@ exports.onCreateDevServer = async ( const job = queue.get(req.originalUrl) // wait until the file has been processed and saved to disk - await scheduleJob(job, boundActionCreators, pluginOptions) + await scheduleJob(job, boundActionCreators, pluginOptions, false) // remove job from queue because it has been processed queue.delete(req.originalUrl) diff --git a/packages/gatsby-plugin-sharp/src/scheduler.js b/packages/gatsby-plugin-sharp/src/scheduler.js index 16697bdc6ed9f..8d296652b78ed 100644 --- a/packages/gatsby-plugin-sharp/src/scheduler.js +++ b/packages/gatsby-plugin-sharp/src/scheduler.js @@ -18,7 +18,12 @@ const bar = new ProgressBar( } ) -exports.scheduleJob = async (job, boundActionCreators, pluginOptions) => { +exports.scheduleJob = async ( + job, + boundActionCreators, + pluginOptions, + reportStatus = true +) => { const inputFileKey = job.inputPath.replace(/\./g, `%2E`) const outputFileKey = job.outputPath.replace(/\./g, `%2E`) const jobPath = `${inputFileKey}.${outputFileKey}` @@ -55,14 +60,26 @@ exports.scheduleJob = async (job, boundActionCreators, pluginOptions) => { if (!isQueued) { q.push(cb => { - runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) + runJobs( + inputFileKey, + boundActionCreators, + pluginOptions, + reportStatus, + cb + ) }) } return deferred.promise } -function runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) { +function runJobs( + inputFileKey, + boundActionCreators, + pluginOptions, + reportStatus, + cb +) { const jobs = _.values(toProcess[inputFileKey]) const findDeferred = job => jobs.find(j => j.job === job).deferred const { job } = jobs[0] @@ -99,7 +116,12 @@ function runJobs(inputFileKey, boundActionCreators, pluginOptions, cb) { }) .then(() => { imagesFinished += 1 - bar.tick() + + // only show progress on build + if (reportStatus) { + bar.tick() + } + boundActionCreators.setJob( { id: `processing image ${job.inputPath}`, From 5c7d6f9ec956fe6fe7f9f254ece75c37e534e6e5 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Feb 2019 01:50:21 -0800 Subject: [PATCH 21/36] update snapshots with jest-path-serializer --- jest.config.js | 1 + package.json | 1 + .../src/__tests__/__snapshots__/index.js.snap | 4 ++-- .../__tests__/__snapshots__/normalize.js.snap | 4 ++-- .../__tests__/__snapshots__/gatsby-node.js.snap | 2 +- .../__tests__/__snapshots__/gatsby-node.js.snap | 6 +++--- .../__tests__/__snapshots__/gatsby-node.js.snap | 2 +- .../__tests__/__snapshots__/gatsby-node.js.snap | 2 +- yarn.lock | 16 +++++++++++++++- 9 files changed, 27 insertions(+), 11 deletions(-) diff --git a/jest.config.js b/jest.config.js index 65ac17ebc84e6..b8e4f7ed6aa76 100644 --- a/jest.config.js +++ b/jest.config.js @@ -32,6 +32,7 @@ module.exports = { moduleNameMapper: { "^highlight.js$": `/node_modules/highlight.js/lib/index.js`, }, + snapshotSerializers: [`jest-serializer-path`], collectCoverage: useCoverage, coverageReporters: [`json-summary`, `text`, `html`, `cobertura`], coverageThreshold: { diff --git a/package.json b/package.json index 9873a65b39793..60db81d1f01d5 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "jest": "^24.0.0", "jest-cli": "^24.0.0", "jest-junit": "^6.1.0", + "jest-serializer-path": "^0.1.15", "lerna": "^3.10.7", "lint-staged": "^8.0.4", "markdown-magic": "^0.1.25", diff --git a/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap b/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap index 7d0833ef8be69..783ca0844ca4e 100644 --- a/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap +++ b/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap @@ -90,8 +90,8 @@ exports[`gatsby-plugin-sharp queueImageResizing should process immediately when "toFormat": "png", "width": 3, }, - "inputPath": "/home/ward/projects/os/gatsby/packages/gatsby-plugin-sharp/src/__tests__/images/144-density.png", - "outputPath": "/home/ward/projects/os/gatsby/public/static/1234/39ca0/test.png", + "inputPath": "/packages/gatsby-plugin-sharp/src/__tests__/images/144-density.png", + "outputPath": "/public/static/1234/39ca0/test.png", "src": "/static/1234/39ca0/test.png", }, Object { diff --git a/packages/gatsby-source-contentful/src/__tests__/__snapshots__/normalize.js.snap b/packages/gatsby-source-contentful/src/__tests__/__snapshots__/normalize.js.snap index fbac90a1e5c5f..b192da000c1a9 100644 --- a/packages/gatsby-source-contentful/src/__tests__/__snapshots__/normalize.js.snap +++ b/packages/gatsby-source-contentful/src/__tests__/__snapshots__/normalize.js.snap @@ -3963,7 +3963,7 @@ Unsere Lemnos Produkte werden sorgfältig von unseren Handwerkern fein geschliff ], "id": "uuid-from-gatsby", "internal": Object { - "content": "{\\"engines\\":{\\"yarn\\":\\"^1.2.1\\"},\\"private\\":true,\\"scripts\\":{\\"jest\\":\\"jest\\",\\"lint\\":\\"eslint --ext .js,.jsx packages/**/src\\",\\"plop\\":\\"plop\\",\\"test\\":\\"yarn lint && yarn jest\\",\\"lerna\\":\\"lerna\\",\\"watch\\":\\"lerna run watch --no-sort --stream --concurrency 999\\",\\"format\\":\\"npm run format-packages && npm run format-cache-dir && npm run format-www && npm run format-examples && npm run format-scripts\\",\\"publish\\":\\"lerna publish\\",\\"bootstrap\\":\\"yarn && npm run check-versions && lerna run prepare\\",\\"lint:flow\\":\\"babel-node scripts/flow-check.js\\",\\"remotedev\\":\\"remotedev --hostname=localhost --port=19999\\",\\"test_bkup\\":\\"npm run lint && npm run test-node && npm run test-integration\\",\\"format-www\\":\\"prettier-eslint --write \\\\\\"www/*.js\\\\\\" \\\\\\"www/src/**/*.js\\\\\\"\\",\\"test:watch\\":\\"jest --watch\\",\\"test:update\\":\\"jest --updateSnapshot\\",\\"publish-next\\":\\"lerna publish --npm-tag=next\\",\\"check-versions\\":\\"babel-node scripts/check-versions.js\\",\\"format-scripts\\":\\"prettier-eslint --write \\\\\\"scripts/**/*.js\\\\\\"\\",\\"publish-canary\\":\\"lerna publish --canary --yes\\",\\"format-examples\\":\\"prettier-eslint --write \\\\\\"examples/**/gatsby-node.js\\\\\\" \\\\\\"examples/**/gatsby-config.js\\\\\\" \\\\\\"examples/**/src/**/*.js\\\\\\"\\",\\"format-packages\\":\\"prettier-eslint --write \\\\\\"packages/*/src/**/*.js\\\\\\"\\",\\"format-cache-dir\\":\\"prettier-eslint --write \\\\\\"packages/gatsby/cache-dir/*.js\\\\\\"\\"},\\"workspaces\\":[\\"packages/*\\"],\\"eslintIgnore\\":[\\"interfaces\\",\\"**/__tests__/fixtures/\\"],\\"devDependencies\\":{\\"glob\\":\\"^7.1.1\\",\\"jest\\":\\"^20.0.4\\",\\"plop\\":\\"^1.8.1\\",\\"lerna\\":\\"^2.1.1\\",\\"eslint\\":\\"^4.5.0\\",\\"rimraf\\":\\"^2.6.1\\",\\"chokidar\\":\\"^1.7.0\\",\\"flow-bin\\":\\"^0.42.0\\",\\"jest-cli\\":\\"^20.0.4\\",\\"prettier\\":\\"^1.7.0\\",\\"babel-cli\\":\\"^6.26.0\\",\\"cross-env\\":\\"^5.0.5\\",\\"babel-jest\\":\\"^20.0.3\\",\\"babel-eslint\\":\\"^7.2.3\\",\\"babel-runtime\\":\\"^6.26.0\\",\\"babel-register\\":\\"^6.26.0\\",\\"babel-preset-env\\":\\"^1.6.0\\",\\"remotedev-server\\":\\"^0.2.3\\",\\"babel-preset-flow\\":\\"^6.23.0\\",\\"babel-preset-react\\":\\"^6.24.1\\",\\"babel-plugin-lodash\\":\\"^3.2.11\\",\\"eslint-plugin-react\\":\\"^7.3.0\\",\\"prettier-eslint-cli\\":\\"4.2.x\\",\\"babel-preset-stage-0\\":\\"^6.24.1\\",\\"eslint-config-google\\":\\"^0.9.1\\",\\"eslint-plugin-import\\":\\"^2.7.0\\",\\"eslint-config-prettier\\":\\"^2.5.0\\",\\"eslint-plugin-flowtype\\":\\"^2.35.0\\",\\"eslint-plugin-jsx-a11y\\":\\"^6.0.2\\",\\"eslint-plugin-prettier\\":\\"^2.2.0\\",\\"babel-plugin-transform-runtime\\":\\"^6.23.0\\",\\"babel-plugin-add-module-exports\\":\\"^0.2.1\\",\\"babel-plugin-transform-flow-strip-types\\":\\"^6.22.0\\",\\"babel-plugin-transform-async-to-generator\\":\\"^6.24.1\\"}}", + "content": "{\\"engines\\":{\\"yarn\\":\\"^1.2.1\\"},\\"private\\":true,\\"scripts\\":{\\"jest\\":\\"jest\\",\\"lint\\":\\"eslint --ext .js,.jsx packages/**/src\\",\\"plop\\":\\"plop\\",\\"test\\":\\"yarn lint && yarn jest\\",\\"lerna\\":\\"lerna\\",\\"watch\\":\\"lerna run watch --no-sort --stream --concurrency 999\\",\\"format\\":\\"npm run format-packages && npm run format-cache-dir && npm run format-www && npm run format-examples && npm run format-scripts\\",\\"publish\\":\\"lerna publish\\",\\"bootstrap\\":\\"yarn && npm run check-versions && lerna run prepare\\",\\"lint:flow\\":\\"babel-node scripts/flow-check.js\\",\\"remotedev\\":\\"remotedev --hostname=localhost --port=19999\\",\\"test_bkup\\":\\"npm run lint && npm run test-node && npm run test-integration\\",\\"format-www\\":\\"prettier-eslint --write /\\"www/*.js/\\" /\\"www/src/**/*.js/\\"\\",\\"test:watch\\":\\"jest --watch\\",\\"test:update\\":\\"jest --updateSnapshot\\",\\"publish-next\\":\\"lerna publish --npm-tag=next\\",\\"check-versions\\":\\"babel-node scripts/check-versions.js\\",\\"format-scripts\\":\\"prettier-eslint --write /\\"scripts/**/*.js/\\"\\",\\"publish-canary\\":\\"lerna publish --canary --yes\\",\\"format-examples\\":\\"prettier-eslint --write /\\"examples/**/gatsby-node.js/\\" /\\"examples/**/gatsby-config.js/\\" /\\"examples/**/src/**/*.js/\\"\\",\\"format-packages\\":\\"prettier-eslint --write /\\"packages/*/src/**/*.js/\\"\\",\\"format-cache-dir\\":\\"prettier-eslint --write /\\"packages/gatsby/cache-dir/*.js/\\"\\"},\\"workspaces\\":[\\"packages/*\\"],\\"eslintIgnore\\":[\\"interfaces\\",\\"**/__tests__/fixtures/\\"],\\"devDependencies\\":{\\"glob\\":\\"^7.1.1\\",\\"jest\\":\\"^20.0.4\\",\\"plop\\":\\"^1.8.1\\",\\"lerna\\":\\"^2.1.1\\",\\"eslint\\":\\"^4.5.0\\",\\"rimraf\\":\\"^2.6.1\\",\\"chokidar\\":\\"^1.7.0\\",\\"flow-bin\\":\\"^0.42.0\\",\\"jest-cli\\":\\"^20.0.4\\",\\"prettier\\":\\"^1.7.0\\",\\"babel-cli\\":\\"^6.26.0\\",\\"cross-env\\":\\"^5.0.5\\",\\"babel-jest\\":\\"^20.0.3\\",\\"babel-eslint\\":\\"^7.2.3\\",\\"babel-runtime\\":\\"^6.26.0\\",\\"babel-register\\":\\"^6.26.0\\",\\"babel-preset-env\\":\\"^1.6.0\\",\\"remotedev-server\\":\\"^0.2.3\\",\\"babel-preset-flow\\":\\"^6.23.0\\",\\"babel-preset-react\\":\\"^6.24.1\\",\\"babel-plugin-lodash\\":\\"^3.2.11\\",\\"eslint-plugin-react\\":\\"^7.3.0\\",\\"prettier-eslint-cli\\":\\"4.2.x\\",\\"babel-preset-stage-0\\":\\"^6.24.1\\",\\"eslint-config-google\\":\\"^0.9.1\\",\\"eslint-plugin-import\\":\\"^2.7.0\\",\\"eslint-config-prettier\\":\\"^2.5.0\\",\\"eslint-plugin-flowtype\\":\\"^2.35.0\\",\\"eslint-plugin-jsx-a11y\\":\\"^6.0.2\\",\\"eslint-plugin-prettier\\":\\"^2.2.0\\",\\"babel-plugin-transform-runtime\\":\\"^6.23.0\\",\\"babel-plugin-add-module-exports\\":\\"^0.2.1\\",\\"babel-plugin-transform-flow-strip-types\\":\\"^6.22.0\\",\\"babel-plugin-transform-async-to-generator\\":\\"^6.24.1\\"}}", "contentDigest": "bd3309e9154a040413fe8717b19fd4d3", "mediaType": "application/json", "type": "contentfulJsonTestJsonTestJsonNode", @@ -4079,7 +4079,7 @@ Unsere Lemnos Produkte werden sorgfältig von unseren Handwerkern fein geschliff ], "id": "uuid-from-gatsby", "internal": Object { - "content": "{\\"engines\\":{\\"yarn\\":\\"^1.2.1\\"},\\"private\\":true,\\"scripts\\":{\\"jest\\":\\"jest\\",\\"lint\\":\\"eslint --ext .js,.jsx packages/**/src\\",\\"plop\\":\\"plop\\",\\"test\\":\\"yarn lint && yarn jest\\",\\"lerna\\":\\"lerna\\",\\"watch\\":\\"lerna run watch --no-sort --stream --concurrency 999\\",\\"format\\":\\"npm run format-packages && npm run format-cache-dir && npm run format-www && npm run format-examples && npm run format-scripts\\",\\"publish\\":\\"lerna publish\\",\\"bootstrap\\":\\"yarn && npm run check-versions && lerna run prepare\\",\\"lint:flow\\":\\"babel-node scripts/flow-check.js\\",\\"remotedev\\":\\"remotedev --hostname=localhost --port=19999\\",\\"test_bkup\\":\\"npm run lint && npm run test-node && npm run test-integration\\",\\"format-www\\":\\"prettier-eslint --write \\\\\\"www/*.js\\\\\\" \\\\\\"www/src/**/*.js\\\\\\"\\",\\"test:watch\\":\\"jest --watch\\",\\"test:update\\":\\"jest --updateSnapshot\\",\\"publish-next\\":\\"lerna publish --npm-tag=next\\",\\"check-versions\\":\\"babel-node scripts/check-versions.js\\",\\"format-scripts\\":\\"prettier-eslint --write \\\\\\"scripts/**/*.js\\\\\\"\\",\\"publish-canary\\":\\"lerna publish --canary --yes\\",\\"format-examples\\":\\"prettier-eslint --write \\\\\\"examples/**/gatsby-node.js\\\\\\" \\\\\\"examples/**/gatsby-config.js\\\\\\" \\\\\\"examples/**/src/**/*.js\\\\\\"\\",\\"format-packages\\":\\"prettier-eslint --write \\\\\\"packages/*/src/**/*.js\\\\\\"\\",\\"format-cache-dir\\":\\"prettier-eslint --write \\\\\\"packages/gatsby/cache-dir/*.js\\\\\\"\\"},\\"workspaces\\":[\\"packages/*\\"],\\"eslintIgnore\\":[\\"interfaces\\",\\"**/__tests__/fixtures/\\"],\\"devDependencies\\":{\\"glob\\":\\"^7.1.1\\",\\"jest\\":\\"^20.0.4\\",\\"plop\\":\\"^1.8.1\\",\\"lerna\\":\\"^2.1.1\\",\\"eslint\\":\\"^4.5.0\\",\\"rimraf\\":\\"^2.6.1\\",\\"chokidar\\":\\"^1.7.0\\",\\"flow-bin\\":\\"^0.42.0\\",\\"jest-cli\\":\\"^20.0.4\\",\\"prettier\\":\\"^1.7.0\\",\\"babel-cli\\":\\"^6.26.0\\",\\"cross-env\\":\\"^5.0.5\\",\\"babel-jest\\":\\"^20.0.3\\",\\"babel-eslint\\":\\"^7.2.3\\",\\"babel-runtime\\":\\"^6.26.0\\",\\"babel-register\\":\\"^6.26.0\\",\\"babel-preset-env\\":\\"^1.6.0\\",\\"remotedev-server\\":\\"^0.2.3\\",\\"babel-preset-flow\\":\\"^6.23.0\\",\\"babel-preset-react\\":\\"^6.24.1\\",\\"babel-plugin-lodash\\":\\"^3.2.11\\",\\"eslint-plugin-react\\":\\"^7.3.0\\",\\"prettier-eslint-cli\\":\\"4.2.x\\",\\"babel-preset-stage-0\\":\\"^6.24.1\\",\\"eslint-config-google\\":\\"^0.9.1\\",\\"eslint-plugin-import\\":\\"^2.7.0\\",\\"eslint-config-prettier\\":\\"^2.5.0\\",\\"eslint-plugin-flowtype\\":\\"^2.35.0\\",\\"eslint-plugin-jsx-a11y\\":\\"^6.0.2\\",\\"eslint-plugin-prettier\\":\\"^2.2.0\\",\\"babel-plugin-transform-runtime\\":\\"^6.23.0\\",\\"babel-plugin-add-module-exports\\":\\"^0.2.1\\",\\"babel-plugin-transform-flow-strip-types\\":\\"^6.22.0\\",\\"babel-plugin-transform-async-to-generator\\":\\"^6.24.1\\"}}", + "content": "{\\"engines\\":{\\"yarn\\":\\"^1.2.1\\"},\\"private\\":true,\\"scripts\\":{\\"jest\\":\\"jest\\",\\"lint\\":\\"eslint --ext .js,.jsx packages/**/src\\",\\"plop\\":\\"plop\\",\\"test\\":\\"yarn lint && yarn jest\\",\\"lerna\\":\\"lerna\\",\\"watch\\":\\"lerna run watch --no-sort --stream --concurrency 999\\",\\"format\\":\\"npm run format-packages && npm run format-cache-dir && npm run format-www && npm run format-examples && npm run format-scripts\\",\\"publish\\":\\"lerna publish\\",\\"bootstrap\\":\\"yarn && npm run check-versions && lerna run prepare\\",\\"lint:flow\\":\\"babel-node scripts/flow-check.js\\",\\"remotedev\\":\\"remotedev --hostname=localhost --port=19999\\",\\"test_bkup\\":\\"npm run lint && npm run test-node && npm run test-integration\\",\\"format-www\\":\\"prettier-eslint --write /\\"www/*.js/\\" /\\"www/src/**/*.js/\\"\\",\\"test:watch\\":\\"jest --watch\\",\\"test:update\\":\\"jest --updateSnapshot\\",\\"publish-next\\":\\"lerna publish --npm-tag=next\\",\\"check-versions\\":\\"babel-node scripts/check-versions.js\\",\\"format-scripts\\":\\"prettier-eslint --write /\\"scripts/**/*.js/\\"\\",\\"publish-canary\\":\\"lerna publish --canary --yes\\",\\"format-examples\\":\\"prettier-eslint --write /\\"examples/**/gatsby-node.js/\\" /\\"examples/**/gatsby-config.js/\\" /\\"examples/**/src/**/*.js/\\"\\",\\"format-packages\\":\\"prettier-eslint --write /\\"packages/*/src/**/*.js/\\"\\",\\"format-cache-dir\\":\\"prettier-eslint --write /\\"packages/gatsby/cache-dir/*.js/\\"\\"},\\"workspaces\\":[\\"packages/*\\"],\\"eslintIgnore\\":[\\"interfaces\\",\\"**/__tests__/fixtures/\\"],\\"devDependencies\\":{\\"glob\\":\\"^7.1.1\\",\\"jest\\":\\"^20.0.4\\",\\"plop\\":\\"^1.8.1\\",\\"lerna\\":\\"^2.1.1\\",\\"eslint\\":\\"^4.5.0\\",\\"rimraf\\":\\"^2.6.1\\",\\"chokidar\\":\\"^1.7.0\\",\\"flow-bin\\":\\"^0.42.0\\",\\"jest-cli\\":\\"^20.0.4\\",\\"prettier\\":\\"^1.7.0\\",\\"babel-cli\\":\\"^6.26.0\\",\\"cross-env\\":\\"^5.0.5\\",\\"babel-jest\\":\\"^20.0.3\\",\\"babel-eslint\\":\\"^7.2.3\\",\\"babel-runtime\\":\\"^6.26.0\\",\\"babel-register\\":\\"^6.26.0\\",\\"babel-preset-env\\":\\"^1.6.0\\",\\"remotedev-server\\":\\"^0.2.3\\",\\"babel-preset-flow\\":\\"^6.23.0\\",\\"babel-preset-react\\":\\"^6.24.1\\",\\"babel-plugin-lodash\\":\\"^3.2.11\\",\\"eslint-plugin-react\\":\\"^7.3.0\\",\\"prettier-eslint-cli\\":\\"4.2.x\\",\\"babel-preset-stage-0\\":\\"^6.24.1\\",\\"eslint-config-google\\":\\"^0.9.1\\",\\"eslint-plugin-import\\":\\"^2.7.0\\",\\"eslint-config-prettier\\":\\"^2.5.0\\",\\"eslint-plugin-flowtype\\":\\"^2.35.0\\",\\"eslint-plugin-jsx-a11y\\":\\"^6.0.2\\",\\"eslint-plugin-prettier\\":\\"^2.2.0\\",\\"babel-plugin-transform-runtime\\":\\"^6.23.0\\",\\"babel-plugin-add-module-exports\\":\\"^0.2.1\\",\\"babel-plugin-transform-flow-strip-types\\":\\"^6.22.0\\",\\"babel-plugin-transform-async-to-generator\\":\\"^6.24.1\\"}}", "contentDigest": "bd3309e9154a040413fe8717b19fd4d3", "mediaType": "application/json", "type": "contentfulJsonTestJsonTestJsonNode", diff --git a/packages/gatsby-transformer-hjson/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-hjson/src/__tests__/__snapshots__/gatsby-node.js.snap index 3b3f9c0945aeb..a10b0dc1966cb 100644 --- a/packages/gatsby-transformer-hjson/src/__tests__/__snapshots__/gatsby-node.js.snap +++ b/packages/gatsby-transformer-hjson/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -40,7 +40,7 @@ Array [ blue: true funny: yup }", - "dir": "/tmp/foo/", + "dir": "/foo/", "id": "whatever", "internal": Object { "contentDigest": "whatever", diff --git a/packages/gatsby-transformer-json/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-json/src/__tests__/__snapshots__/gatsby-node.js.snap index f948c87169a13..694f18ef93349 100644 --- a/packages/gatsby-transformer-json/src/__tests__/__snapshots__/gatsby-node.js.snap +++ b/packages/gatsby-transformer-json/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -36,7 +36,7 @@ Array [ "parent": Object { "children": Array [], "content": "{\\"id\\":\\"foo\\",\\"blue\\":true,\\"funny\\":\\"yup\\"}", - "dir": "/tmp/foo/", + "dir": "/foo/", "id": "whatever", "internal": Object { "contentDigest": "whatever", @@ -149,7 +149,7 @@ Array [ "parent": Object { "children": Array [], "content": "[{\\"id\\":\\"foo\\",\\"blue\\":true,\\"funny\\":\\"yup\\"},{\\"blue\\":false,\\"funny\\":\\"nope\\"}]", - "dir": "/tmp/foo/", + "dir": "/foo/", "id": "whatever", "internal": Object { "contentDigest": "whatever", @@ -177,7 +177,7 @@ Array [ "parent": Object { "children": Array [], "content": "[{\\"id\\":\\"foo\\",\\"blue\\":true,\\"funny\\":\\"yup\\"},{\\"blue\\":false,\\"funny\\":\\"nope\\"}]", - "dir": "/tmp/foo/", + "dir": "/foo/", "id": "whatever", "internal": Object { "contentDigest": "whatever", diff --git a/packages/gatsby-transformer-toml/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-toml/src/__tests__/__snapshots__/gatsby-node.js.snap index b6e6833b5375b..935d10a60bed1 100644 --- a/packages/gatsby-transformer-toml/src/__tests__/__snapshots__/gatsby-node.js.snap +++ b/packages/gatsby-transformer-toml/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -81,7 +81,7 @@ Array [ test_array = [ \\"] \\", \\" # \\"] # ] test_array2 = [ \\"Test #11 ]proved that\\", \\"Experiment #9 was a success\\" ] another_test_string = \\" Same thing, but with a string #\\" - harder_test_string = \\" And when \\\\\\"'s are in the string, along with # \\\\\\"\\" + harder_test_string = \\" And when /\\"'s are in the string, along with # /\\"\\" # Things will get harder [the.hard.\\"bit#\\"] diff --git a/packages/gatsby-transformer-yaml/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-yaml/src/__tests__/__snapshots__/gatsby-node.js.snap index 1947753170918..283e0a5676662 100644 --- a/packages/gatsby-transformer-yaml/src/__tests__/__snapshots__/gatsby-node.js.snap +++ b/packages/gatsby-transformer-yaml/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -38,7 +38,7 @@ Array [ "content": "blue: true funny: yup ", - "dir": "/tmp/bar/", + "dir": "/bar/", "id": "whatever", "internal": Object { "contentDigest": "whatever", diff --git a/yarn.lock b/yarn.lock index 8dfbab0309055..34bb778fa22d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11328,6 +11328,15 @@ jest-runtime@^24.0.0: write-file-atomic "^2.4.2" yargs "^12.0.2" +jest-serializer-path@^0.1.15: + version "0.1.15" + resolved "https://registry.yarnpkg.com/jest-serializer-path/-/jest-serializer-path-0.1.15.tgz#68528c98e3f8be652484f1c3876228f6864bd662" + integrity sha512-sXb+Ckz9LK5bB5sRAXFeG/nwhGn1XBKayy9xhPpgOmAWaljJiS+VN/xJ3P4I19jZ+WejowvO/kES+A9HePTMvQ== + dependencies: + lodash.clonedeep "^4.5.0" + lodash.iserror "^3.1.1" + slash "^2.0.0" + jest-serializer@^24.0.0: version "24.0.0" resolved "https://registry.yarnpkg.com/jest-serializer/-/jest-serializer-24.0.0.tgz#522c44a332cdd194d8c0531eb06a1ee5afb4256b" @@ -12412,6 +12421,11 @@ lodash.isarray@^3.0.0: resolved "https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-3.0.4.tgz#79e4eb88c36a8122af86f844aa9bcd851b5fbb55" integrity sha1-eeTriMNqgSKvhvhEqpvNhRtfu1U= +lodash.iserror@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/lodash.iserror/-/lodash.iserror-3.1.1.tgz#297b9a05fab6714bc2444d7cc19d1d7c44b5ecec" + integrity sha1-KXuaBfq2cUvCRE18wZ0dfES17Ow= + lodash.isnumber@^3.0.0: version "3.0.3" resolved "https://registry.yarnpkg.com/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz#3ce76810c5928d03352301ac287317f11c0b1ffc" @@ -20002,7 +20016,7 @@ webpack@^4.14.0, webpack@^4.16.0: watchpack "^1.5.0" webpack-sources "^1.2.0" -webpack@~4.28.2: +webpack@~4.28.4: version "4.28.4" resolved "https://registry.yarnpkg.com/webpack/-/webpack-4.28.4.tgz#1ddae6c89887d7efb752adf0c3cd32b9b07eacd0" integrity sha512-NxjD61WsK/a3JIdwWjtIpimmvE6UrRi3yG54/74Hk9rwNj5FPkA4DJCf1z4ByDWLkvZhTZE+P3C/eh6UD5lDcw== From f48c6a22a4918ab004e188b15c1ec7c4422b5280 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Feb 2019 03:02:58 -0800 Subject: [PATCH 22/36] fix workflow --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3fcc7a76ffe50..9b630ded9c961 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -212,9 +212,9 @@ workflows: requires: - bootstrap - integration_tests_long_term_caching: - <<: *ignore_docs + <<: *e2e-test-workflow - integration_tests_gatsby_pipeline: - <<: *ignore_docs + <<: *e2e-test-workflow - e2e_tests_gatsbygram: <<: *e2e-test-workflow - e2e_tests_path-prefix: From 790850f1f3073708a3c95d130edae4da743a0a32 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Feb 2019 03:22:56 -0800 Subject: [PATCH 23/36] =?UTF-8?q?retrigger=20ci=20=F0=9F=A4=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 73cc15c423530c479d36eea96a9935ff447a2577 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Feb 2019 03:59:45 -0800 Subject: [PATCH 24/36] fix azure tests --- packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js | 3 ++- packages/gatsby-transformer-json/src/__tests__/gatsby-node.js | 3 ++- packages/gatsby-transformer-yaml/src/__tests__/gatsby-node.js | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js index eccbcb21b434f..4fff3357b6c3c 100644 --- a/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js @@ -1,4 +1,5 @@ const Promise = require(`bluebird`) +const os = require(`os`) const HJSON = require(`hjson`) const { onCreateNode } = require(`../gatsby-node`) @@ -48,7 +49,7 @@ describe(`Process HJSON nodes correctly`, () => { it(`correctly creates a node from HJSON which is a single object`, async () => { const data = { id: `foo`, blue: true, funny: `yup` } node.content = HJSON.stringify(data) - node.dir = `/tmp/foo/` + node.dir = `${os.tmpdir()}/foo/` const createNode = jest.fn() const createParentChildLink = jest.fn() diff --git a/packages/gatsby-transformer-json/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-json/src/__tests__/gatsby-node.js index 196f2f22eca22..0ee81d0bc2b83 100644 --- a/packages/gatsby-transformer-json/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-json/src/__tests__/gatsby-node.js @@ -1,4 +1,5 @@ const Promise = require(`bluebird`) +const os = require(`os`) const { onCreateNode } = require(`../gatsby-node`) @@ -44,7 +45,7 @@ describe(`Process JSON nodes correctly`, () => { const baseFileNode = { ...baseNode, name: `nodeName`, - dir: `/tmp/foo/`, + dir: `${os.tmpdir()}/foo/`, internal: { ...baseNode.internal, type: `File`, diff --git a/packages/gatsby-transformer-yaml/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-yaml/src/__tests__/gatsby-node.js index e9684ab61fda6..350b191300651 100644 --- a/packages/gatsby-transformer-yaml/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-yaml/src/__tests__/gatsby-node.js @@ -1,4 +1,5 @@ const Promise = require(`bluebird`) +const os = require(`os`) const yaml = require(`js-yaml`) const { onCreateNode } = require(`../gatsby-node`) @@ -46,7 +47,7 @@ describe(`Process YAML nodes correctly`, () => { it(`correctly creates a node from JSON which is a single object`, async () => { const data = { blue: true, funny: `yup` } node.content = yaml.safeDump(data) - node.dir = `/tmp/bar/` + node.dir = `${os.tmpdir()}/bar/` const createNode = jest.fn() const createParentChildLink = jest.fn() From b860e3a0aac59bd318d352b5fd2e57aa138bf47e Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 14 Feb 2019 23:13:48 -0800 Subject: [PATCH 25/36] revert www/README.md --- www/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/www/README.md b/www/README.md index 8e44fff478f7d..52e76d76431ab 100644 --- a/www/README.md +++ b/www/README.md @@ -39,3 +39,15 @@ GATSBY_FEEDBACK_KEY_STARTERLIB=ADD_KEY ``` If there's a problem with the feedback widgets, please open an issue in the repo. + +## Running slow build? (Screenshots placeholder) + +If you are not working on starter or site showcase, it might be beneficial to use a placeholder image instead of actual screenshots. It will skip downloading screenshots and generating responsive images for all screenshots and replace them with a placeholder image. + +Add the following env variable to your `.env.development` file to enable placeholder behaviour: + +``` +GATSBY_SCREENSHOT_PLACEHOLDER=true +``` + +For more information checkout [`gatsby-transformer-screenshot` docs](http://www.gatsbyjs.org/packages/gatsby-transformer-screenshot#placeholder-image). From 3b0f6d5192cb36630123ced3a16c96c2286d47e5 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 14 Feb 2019 23:25:07 -0800 Subject: [PATCH 26/36] remove unused src field --- packages/gatsby-plugin-sharp/src/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index eb3fdb513c58c..06885bb420d14 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -190,7 +190,6 @@ function queueImageResizing({ file, args = {}, reporter }) { args: options, inputPath: file.absolutePath, outputPath: filePath, - src: prefixedSrc, } queue.set(prefixedSrc, job) From d678998739099c9487f32623f07ddb33ebec8e3c Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Thu, 14 Feb 2019 23:57:33 -0800 Subject: [PATCH 27/36] convert cache to setPluginStatus --- .../gatsby-plugin-sharp/src/gatsby-node.js | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 1979d060b8bcf..9782fa28666de 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -6,18 +6,18 @@ const { } = require(`./index`) const { scheduleJob } = require(`./scheduler`) -const getQueueFromCache = async cache => { - const rawQueue = await cache.get(`queue`) +const getQueueFromCache = store => { + const pluginStatus = store.getState().status.plugins[`gatsby-plugin-sharp`] - if (!rawQueue) { + if (!pluginStatus || !pluginStatus.queue) { return new Map() } - return new Map(rawQueue) + return new Map(pluginStatus.queue) } -const saveQueueToCache = async (cache, queue) => { - const cachedQueue = await getQueueFromCache(cache) +const saveQueueToCache = async (store, setPluginStatus, queue) => { + const cachedQueue = getQueueFromCache(store) // merge both queues for (const [key, job] of cachedQueue) { @@ -27,7 +27,7 @@ const saveQueueToCache = async (cache, queue) => { } // JSON.stringify doesn't work on an Map so we need to convert it to an array - return cache.set(`queue`, Array.from(queue)) + setPluginStatus({ queue: Array.from(queue) }) } exports.onPreBootstrap = ({ actions }, pluginOptions) => { @@ -38,15 +38,15 @@ exports.onPreBootstrap = ({ actions }, pluginOptions) => { /** * save queue to the cache */ -exports.onPostBootstrap = async ({ cache, store }) => - saveQueueToCache(cache, jobQueue) +exports.onPostBootstrap = ({ actions: { setPluginStatus }, store }) => + saveQueueToCache(store, setPluginStatus, jobQueue) /** * Execute all unprocessed images on gatsby build */ let promises = [] -exports.onPreBuild = async ({ cache, boundActionCreators }, pluginOptions) => { - const cachedQueue = await getQueueFromCache(cache) +exports.onPreBuild = ({ store, boundActionCreators }, pluginOptions) => { + const cachedQueue = getQueueFromCache(store) for (const [, job] of cachedQueue) { promises.push(scheduleJob(job, boundActionCreators, pluginOptions)) @@ -56,9 +56,9 @@ exports.onPreBuild = async ({ cache, boundActionCreators }, pluginOptions) => { /** * wait for all images to be processed */ -exports.onPostBuild = ({ cache, reporter }) => +exports.onPostBuild = ({ actions: { setPluginStatus }, store, reporter }) => Promise.all(promises) - .then(() => cache.set(`queue`, [])) + .then(() => setPluginStatus({ queue: [] })) .catch(({ err, message }) => { reportError(message || err.message, err, reporter) }) @@ -67,13 +67,15 @@ exports.onPostBuild = ({ cache, reporter }) => * Build images on the fly when they are requested by the browser */ exports.onCreateDevServer = async ( - { app, cache, emitter, boundActionCreators }, + { app, emitter, boundActionCreators, actions: { setPluginStatus }, store }, pluginOptions ) => { - emitter.on(`QUERY_QUEUE_DRAINED`, () => saveQueueToCache(cache, jobQueue)) + emitter.on(`QUERY_QUEUE_DRAINED`, () => + saveQueueToCache(store, setPluginStatus, jobQueue) + ) app.use(async (req, res, next) => { - const queue = await getQueueFromCache(cache) + const queue = getQueueFromCache(store) if (!queue.has(req.originalUrl)) { return next() } @@ -85,7 +87,7 @@ exports.onCreateDevServer = async ( // remove job from queue because it has been processed queue.delete(req.originalUrl) - await saveQueueToCache(cache, queue) + saveQueueToCache(store, setPluginStatus, queue) return res.sendFile(job.outputPath) }) From 62dbc33d27d78939e887de68417879fa870cee72 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Fri, 15 Feb 2019 03:07:42 -0800 Subject: [PATCH 28/36] add lazyImageGeneration option --- .../__tests__/lazy-image-build/develop.js | 22 ++++++ .../gatsby-pipeline/gatsby-config-default.js | 19 +++++ .../gatsby-pipeline/gatsby-config-nolazy.js | 24 ++++++ .../gatsby-pipeline/gatsby-config.js | 7 +- .../gatsby-pipeline/utils/create-devserver.js | 11 ++- .../gatsby-plugin-sharp/src/gatsby-node.js | 76 ++++++++++++++----- packages/gatsby-plugin-sharp/src/index.js | 3 + 7 files changed, 140 insertions(+), 22 deletions(-) create mode 100644 integration-tests/gatsby-pipeline/gatsby-config-default.js create mode 100644 integration-tests/gatsby-pipeline/gatsby-config-nolazy.js diff --git a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js index 356dd0fbcf594..0b58ef2df068b 100644 --- a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js +++ b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js @@ -15,11 +15,21 @@ const cleanDirs = () => fs.emptyDir(`${basePath}/.cache`), ]) +const configFiles = { + gatsby: path.join(__dirname, `../../gatsby-config.js`), + default: path.join(__dirname, `../../gatsby-config-default.js`), + nolazy: path.join(__dirname, `../../gatsby-config-nolazy.js`), +} + describe(`Lazy images`, () => { beforeAll(async () => { await cleanDirs() }) + beforeEach(async () => { + await fs.copy(configFiles.default, configFiles.gatsby) + }) + test(`should process images on demand`, async () => { const { kill } = await createDevServer() @@ -62,4 +72,16 @@ describe(`Lazy images`, () => { const images = glob.sync(`${basePath}/public/**/*.png`) expect(images.length).toBe(6) }) + + test(`should process all images on develop when lazyImageGeneration is false`, async () => { + await fs.copy(configFiles.nolazy, configFiles.gatsby) + + await cleanDirs() + const { kill } = await createDevServer() + + await kill() + + const images = glob.sync(`${basePath}/public/**/*.png`) + expect(images.length).toBe(6) + }) }) diff --git a/integration-tests/gatsby-pipeline/gatsby-config-default.js b/integration-tests/gatsby-pipeline/gatsby-config-default.js new file mode 100644 index 0000000000000..bfda48c3cf314 --- /dev/null +++ b/integration-tests/gatsby-pipeline/gatsby-config-default.js @@ -0,0 +1,19 @@ +module.exports = { + siteMetadata: { + title: `Gatsby Default Starter`, + description: `Kick off your next, great Gatsby project with this default starter. This barebones starter ships with the main Gatsby configuration files you might need.`, + author: `@gatsbyjs`, + }, + plugins: [ + `gatsby-plugin-react-helmet`, + { + resolve: `gatsby-source-filesystem`, + options: { + name: `images`, + path: `${__dirname}/src/images`, + }, + }, + `gatsby-transformer-sharp`, + `gatsby-plugin-sharp`, + ], +} diff --git a/integration-tests/gatsby-pipeline/gatsby-config-nolazy.js b/integration-tests/gatsby-pipeline/gatsby-config-nolazy.js new file mode 100644 index 0000000000000..5baca117a8f67 --- /dev/null +++ b/integration-tests/gatsby-pipeline/gatsby-config-nolazy.js @@ -0,0 +1,24 @@ +module.exports = { + siteMetadata: { + title: `Gatsby Default Starter`, + description: `Kick off your next, great Gatsby project with this default starter. This barebones starter ships with the main Gatsby configuration files you might need.`, + author: `@gatsbyjs`, + }, + plugins: [ + `gatsby-plugin-react-helmet`, + { + resolve: `gatsby-source-filesystem`, + options: { + name: `images`, + path: `${__dirname}/src/images`, + }, + }, + `gatsby-transformer-sharp`, + { + resolve: `gatsby-plugin-sharp`, + options: { + lazyImageGeneration: false, + }, + }, + ], +} diff --git a/integration-tests/gatsby-pipeline/gatsby-config.js b/integration-tests/gatsby-pipeline/gatsby-config.js index bfda48c3cf314..5baca117a8f67 100644 --- a/integration-tests/gatsby-pipeline/gatsby-config.js +++ b/integration-tests/gatsby-pipeline/gatsby-config.js @@ -14,6 +14,11 @@ module.exports = { }, }, `gatsby-transformer-sharp`, - `gatsby-plugin-sharp`, + { + resolve: `gatsby-plugin-sharp`, + options: { + lazyImageGeneration: false, + }, + }, ], } diff --git a/integration-tests/gatsby-pipeline/utils/create-devserver.js b/integration-tests/gatsby-pipeline/utils/create-devserver.js index 81ae0168b8fe2..c7968eecd7857 100644 --- a/integration-tests/gatsby-pipeline/utils/create-devserver.js +++ b/integration-tests/gatsby-pipeline/utils/create-devserver.js @@ -16,6 +16,14 @@ const killProcess = devProcess => process.kill(-devProcess.pid) }) +// messages that tell us to stop proceeding and should resolve the process to be "done" +// so we can stop the test sooner rather than to wait for a timeout +const readyMessages = [ + `You can now view`, + `Failed to compile`, + `Something is already running at`, +] + module.exports = () => new Promise(resolve => { const devProcess = execa(`yarn`, [`develop`], { @@ -25,7 +33,8 @@ module.exports = () => }) devProcess.stdout.on(`data`, chunk => { - if (chunk.toString().includes(`You can now view`)) { + const matches = readyMessages.some(msg => chunk.toString().includes(msg)) + if (matches) { // We only need to expose a kill function, the rest is not needed resolve({ kill: () => killProcess(devProcess) }) } diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index 9782fa28666de..e48f1edcb7605 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -5,6 +5,7 @@ const { reportError, } = require(`./index`) const { scheduleJob } = require(`./scheduler`) +let normalizedOptions = {} const getQueueFromCache = store => { const pluginStatus = store.getState().status.plugins[`gatsby-plugin-sharp`] @@ -30,46 +31,81 @@ const saveQueueToCache = async (store, setPluginStatus, queue) => { setPluginStatus({ queue: Array.from(queue) }) } +const processQueue = (store, boundActionCreators, options) => { + const cachedQueue = getQueueFromCache(store) + + const promises = [] + for (const [, job] of cachedQueue) { + promises.push(scheduleJob(job, boundActionCreators, options)) + } + + return promises +} + +const cleanupQueueAfterProcess = (imageJobs, setPluginStatus, reporter) => + Promise.all(imageJobs) + .then(() => setPluginStatus({ queue: [] })) + .catch(({ err, message }) => { + reportError(message || err.message, err, reporter) + }) + exports.onPreBootstrap = ({ actions }, pluginOptions) => { setBoundActionCreators(actions) - setPluginOptions(pluginOptions) + normalizedOptions = setPluginOptions(pluginOptions) } /** - * save queue to the cache + * save queue to the cache or process queue */ -exports.onPostBootstrap = ({ actions: { setPluginStatus }, store }) => +exports.onPostBootstrap = ({ + store, + boundActionCreators, + actions: { setPluginStatus }, + reporter, +}) => { + // Save queue saveQueueToCache(store, setPluginStatus, jobQueue) + if (normalizedOptions.lazyImageGeneration) { + return + } + + const imageJobs = processQueue(store, boundActionCreators, normalizedOptions) + + cleanupQueueAfterProcess(imageJobs, setPluginStatus, reporter) + + return +} + /** * Execute all unprocessed images on gatsby build */ let promises = [] -exports.onPreBuild = ({ store, boundActionCreators }, pluginOptions) => { - const cachedQueue = getQueueFromCache(store) - - for (const [, job] of cachedQueue) { - promises.push(scheduleJob(job, boundActionCreators, pluginOptions)) - } +exports.onPreBuild = ({ store, boundActionCreators }) => { + promises = processQueue(store, boundActionCreators, normalizedOptions) } /** * wait for all images to be processed */ -exports.onPostBuild = ({ actions: { setPluginStatus }, store, reporter }) => - Promise.all(promises) - .then(() => setPluginStatus({ queue: [] })) - .catch(({ err, message }) => { - reportError(message || err.message, err, reporter) - }) +exports.onPostBuild = ({ actions: { setPluginStatus }, reporter }) => + cleanupQueueAfterProcess(promises, setPluginStatus, reporter) /** * Build images on the fly when they are requested by the browser */ -exports.onCreateDevServer = async ( - { app, emitter, boundActionCreators, actions: { setPluginStatus }, store }, - pluginOptions -) => { +exports.onCreateDevServer = async ({ + app, + emitter, + boundActionCreators, + actions: { setPluginStatus }, + store, +}) => { + // no need to do set things up when people opt out + if (!normalizedOptions.lazyImageGeneration) { + return + } + emitter.on(`QUERY_QUEUE_DRAINED`, () => saveQueueToCache(store, setPluginStatus, jobQueue) ) @@ -83,7 +119,7 @@ exports.onCreateDevServer = async ( const job = queue.get(req.originalUrl) // wait until the file has been processed and saved to disk - await scheduleJob(job, boundActionCreators, pluginOptions, false) + await scheduleJob(job, boundActionCreators, normalizedOptions, false) // remove job from queue because it has been processed queue.delete(req.originalUrl) diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index 06885bb420d14..45f7a53bb6311 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -42,10 +42,13 @@ exports.queue = queue const pluginDefaults = { useMozJpeg: process.env.GATSBY_JPEG_ENCODER === `MOZJPEG`, stripMetadata: true, + lazyImageGeneration: true, } let pluginOptions = Object.assign({}, pluginDefaults) exports.setPluginOptions = opts => { pluginOptions = Object.assign({}, pluginOptions, opts) + + return pluginOptions } const reportError = (message, err, reporter) => { From 401add3fd876784cd26cc8781f782c3186f3a1b8 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Fri, 15 Feb 2019 03:37:35 -0800 Subject: [PATCH 29/36] fix yarn --- yarn.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index a02d3078f47a5..3a0a659a612b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -983,10 +983,10 @@ resolved "https://registry.yarnpkg.com/@emotion/utils/-/utils-0.11.1.tgz#8529b7412a6eb4b48bdf6e720cc1b8e6e1e17628" integrity sha512-8M3VN0hetwhsJ8dH8VkVy7xo5/1VoBsDOk/T4SJOeXwTO1c4uIqVNx2qyecLFnnUWD5vvUqHQ1gASSeUN6zcTg== -"@gatsbyjs/relay-compiler@2.0.0-printer-fix": - version "2.0.0-printer-fix" - resolved "https://registry.yarnpkg.com/@gatsbyjs/relay-compiler/-/relay-compiler-2.0.0-printer-fix.tgz#542b3eb8e7f28142ccbde051dfc47c6ce506d022" - integrity sha512-oxUgELwerXuQWQhQz+G3xXAP9H4uMgj8aNGmNp0IYKv9D0puvUw+W6884jae57Ng+Vj2K5rxUkn0ApivSTvutQ== +"@gatsbyjs/relay-compiler@2.0.0-printer-fix.2": + version "2.0.0-printer-fix.2" + resolved "https://registry.yarnpkg.com/@gatsbyjs/relay-compiler/-/relay-compiler-2.0.0-printer-fix.2.tgz#214db0e6072d40ea78ad5fabdb49d56bc95f4e99" + integrity sha512-7GeCCEQ7O15lMTT/SXy9HuRde4cv5vs465ZnLK2QCajSDLior+20yrMqHn1PGsJYK6nNZH7p3lw9qTCpqmuc7Q== dependencies: "@babel/generator" "^7.0.0" "@babel/parser" "^7.0.0" From 03d4e7cbd2dbdf3adf576c202c39ef8f9b024a3d Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Fri, 15 Feb 2019 03:50:36 -0800 Subject: [PATCH 30/36] update snapshots --- .../src/__tests__/__snapshots__/index.js.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap b/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap index 783ca0844ca4e..ab948e1c0b57b 100644 --- a/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap +++ b/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap @@ -92,7 +92,6 @@ exports[`gatsby-plugin-sharp queueImageResizing should process immediately when }, "inputPath": "/packages/gatsby-plugin-sharp/src/__tests__/images/144-density.png", "outputPath": "/public/static/1234/39ca0/test.png", - "src": "/static/1234/39ca0/test.png", }, Object { "addThirdPartySchema": [Function], @@ -120,6 +119,7 @@ exports[`gatsby-plugin-sharp queueImageResizing should process immediately when "touchNode": [Function], }, Object { + "lazyImageGeneration": true, "stripMetadata": true, "useMozJpeg": false, }, From 13cb9b03861729fef08557a7e3cfca2dd42e8480 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 26 Feb 2019 00:50:56 -0800 Subject: [PATCH 31/36] fix image url in test --- .../gatsby-pipeline/__tests__/lazy-image-build/develop.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js index 0b58ef2df068b..36388781f6c5a 100644 --- a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js +++ b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js @@ -34,7 +34,7 @@ describe(`Lazy images`, () => { const { kill } = await createDevServer() const response = await request( - `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a484e/gatsby-astronaut.png`, + `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/cc8d4/gatsby-astronaut.png`, { resolveWithFullResponse: true, } From 11c73336064a941b8eccb99424493a74e43be61f Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 26 Feb 2019 00:51:31 -0800 Subject: [PATCH 32/36] move develop.js to index.js --- .../__tests__/lazy-image-build/{develop.js => index.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename integration-tests/gatsby-pipeline/__tests__/lazy-image-build/{develop.js => index.js} (100%) diff --git a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/index.js similarity index 100% rename from integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js rename to integration-tests/gatsby-pipeline/__tests__/lazy-image-build/index.js From 14227524241edd0a64838f70c5e15e6fe45a60f2 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Wed, 27 Feb 2019 14:13:34 +0100 Subject: [PATCH 33/36] revert tests --- .../__tests__/lazy-image-build/develop.js | 61 +++++++++++++++++++ .../gatsby-pipeline/gatsby-config.js | 7 +-- .../gatsby-pipeline/utils/create-devserver.js | 11 +--- 3 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js diff --git a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js new file mode 100644 index 0000000000000..f0f4f8eea47b6 --- /dev/null +++ b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js @@ -0,0 +1,61 @@ +const execa = require(`execa`) +const path = require(`path`) +const fs = require(`fs-extra`) +const glob = require(`glob`) +const request = require(`request-promise-native`) +const createDevServer = require(`../../utils/create-devserver`) +const basePath = path.resolve(__dirname, `../../`) + +// 2 min +jest.setTimeout(2000 * 60) + +const cleanDirs = () => + Promise.all([ + fs.emptyDir(`${basePath}/public`), + fs.emptyDir(`${basePath}/.cache`), + ]) + +describe(`Lazy images`, () => { + beforeAll(async () => { + await cleanDirs() + }) + + test(`should process images on demand`, async () => { + const { kill } = await createDevServer() + + const response = await request( + `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a2541/gatsby-astronaut.png`, + { + resolveWithFullResponse: true, + } + ) + + await kill() + + expect(response.statusCode).toBe(200) + + const images = glob.sync(`${basePath}/public/**/*.png`) + expect(images.length).toBe(6) + }) + + test(`should process the rest of images on build`, async () => { + await execa(`yarn`, [`build`], { + cwd: basePath, + env: { NODE_ENV: `production` }, + }) + + const images = glob.sync(`${basePath}/public/**/*.png`) + expect(images.length).toBe(6) + }) + + test(`should process all images from a clean build`, async () => { + await cleanDirs() + await execa(`yarn`, [`build`], { + cwd: basePath, + env: { NODE_ENV: `production` }, + }) + + const images = glob.sync(`${basePath}/public/**/*.png`) + expect(images.length).toBe(6) + }) +}) diff --git a/integration-tests/gatsby-pipeline/gatsby-config.js b/integration-tests/gatsby-pipeline/gatsby-config.js index 5baca117a8f67..bfda48c3cf314 100644 --- a/integration-tests/gatsby-pipeline/gatsby-config.js +++ b/integration-tests/gatsby-pipeline/gatsby-config.js @@ -14,11 +14,6 @@ module.exports = { }, }, `gatsby-transformer-sharp`, - { - resolve: `gatsby-plugin-sharp`, - options: { - lazyImageGeneration: false, - }, - }, + `gatsby-plugin-sharp`, ], } diff --git a/integration-tests/gatsby-pipeline/utils/create-devserver.js b/integration-tests/gatsby-pipeline/utils/create-devserver.js index c7968eecd7857..81ae0168b8fe2 100644 --- a/integration-tests/gatsby-pipeline/utils/create-devserver.js +++ b/integration-tests/gatsby-pipeline/utils/create-devserver.js @@ -16,14 +16,6 @@ const killProcess = devProcess => process.kill(-devProcess.pid) }) -// messages that tell us to stop proceeding and should resolve the process to be "done" -// so we can stop the test sooner rather than to wait for a timeout -const readyMessages = [ - `You can now view`, - `Failed to compile`, - `Something is already running at`, -] - module.exports = () => new Promise(resolve => { const devProcess = execa(`yarn`, [`develop`], { @@ -33,8 +25,7 @@ module.exports = () => }) devProcess.stdout.on(`data`, chunk => { - const matches = readyMessages.some(msg => chunk.toString().includes(msg)) - if (matches) { + if (chunk.toString().includes(`You can now view`)) { // We only need to expose a kill function, the rest is not needed resolve({ kill: () => killProcess(devProcess) }) } From 87981e8adf3d68144b31fa2bf805dd506da9a095 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Wed, 27 Feb 2019 14:17:19 +0100 Subject: [PATCH 34/36] comment out all the lazy stuff --- .../gatsby-plugin-sharp/src/gatsby-node.js | 176 +++++++++--------- 1 file changed, 91 insertions(+), 85 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index e48f1edcb7605..d92fc977bc91e 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -2,52 +2,52 @@ const { setBoundActionCreators, setPluginOptions, queue: jobQueue, - reportError, + // reportError, } = require(`./index`) const { scheduleJob } = require(`./scheduler`) let normalizedOptions = {} -const getQueueFromCache = store => { - const pluginStatus = store.getState().status.plugins[`gatsby-plugin-sharp`] +// const getQueueFromCache = store => { +// const pluginStatus = store.getState().status.plugins[`gatsby-plugin-sharp`] - if (!pluginStatus || !pluginStatus.queue) { - return new Map() - } +// if (!pluginStatus || !pluginStatus.queue) { +// return new Map() +// } - return new Map(pluginStatus.queue) -} +// return new Map(pluginStatus.queue) +// } -const saveQueueToCache = async (store, setPluginStatus, queue) => { - const cachedQueue = getQueueFromCache(store) +// const saveQueueToCache = async (store, setPluginStatus, queue) => { +// const cachedQueue = getQueueFromCache(store) - // merge both queues - for (const [key, job] of cachedQueue) { - if (!queue.has(key)) { - queue.set(key, job) - } - } +// // merge both queues +// for (const [key, job] of cachedQueue) { +// if (!queue.has(key)) { +// queue.set(key, job) +// } +// } - // JSON.stringify doesn't work on an Map so we need to convert it to an array - setPluginStatus({ queue: Array.from(queue) }) -} +// // JSON.stringify doesn't work on an Map so we need to convert it to an array +// setPluginStatus({ queue: Array.from(queue) }) +// } -const processQueue = (store, boundActionCreators, options) => { - const cachedQueue = getQueueFromCache(store) +// const processQueue = (store, boundActionCreators, options) => { +// const cachedQueue = getQueueFromCache(store) - const promises = [] - for (const [, job] of cachedQueue) { - promises.push(scheduleJob(job, boundActionCreators, options)) - } +// const promises = [] +// for (const [, job] of cachedQueue) { +// promises.push(scheduleJob(job, boundActionCreators, options)) +// } - return promises -} +// return promises +// } -const cleanupQueueAfterProcess = (imageJobs, setPluginStatus, reporter) => - Promise.all(imageJobs) - .then(() => setPluginStatus({ queue: [] })) - .catch(({ err, message }) => { - reportError(message || err.message, err, reporter) - }) +// const cleanupQueueAfterProcess = (imageJobs, setPluginStatus, reporter) => +// Promise.all(imageJobs) +// .then(() => setPluginStatus({ queue: [] })) +// .catch(({ err, message }) => { +// reportError(message || err.message, err, reporter) +// }) exports.onPreBootstrap = ({ actions }, pluginOptions) => { setBoundActionCreators(actions) @@ -58,76 +58,82 @@ exports.onPreBootstrap = ({ actions }, pluginOptions) => { * save queue to the cache or process queue */ exports.onPostBootstrap = ({ - store, + // store, boundActionCreators, - actions: { setPluginStatus }, - reporter, + // actions: { setPluginStatus }, + // reporter, }) => { - // Save queue - saveQueueToCache(store, setPluginStatus, jobQueue) - - if (normalizedOptions.lazyImageGeneration) { - return + const promises = [] + for (const [, job] of jobQueue) { + promises.push(scheduleJob(job, boundActionCreators, normalizedOptions)) } - const imageJobs = processQueue(store, boundActionCreators, normalizedOptions) + return promises + // // Save queue + // saveQueueToCache(store, setPluginStatus, jobQueue) + + // if (normalizedOptions.lazyImageGeneration) { + // return + // } - cleanupQueueAfterProcess(imageJobs, setPluginStatus, reporter) + // const imageJobs = processQueue(store, boundActionCreators, normalizedOptions) - return + // cleanupQueueAfterProcess(imageJobs, setPluginStatus, reporter) + + // return } /** * Execute all unprocessed images on gatsby build */ -let promises = [] -exports.onPreBuild = ({ store, boundActionCreators }) => { - promises = processQueue(store, boundActionCreators, normalizedOptions) -} +// let promises = [] +// exports.onPreBuild = ({ store, boundActionCreators }) => { +// promises = processQueue(store, boundActionCreators, normalizedOptions) +// } /** * wait for all images to be processed - */ -exports.onPostBuild = ({ actions: { setPluginStatus }, reporter }) => - cleanupQueueAfterProcess(promises, setPluginStatus, reporter) +// */ +// exports.onPostBuild = ({ actions: { setPluginStatus }, reporter }) => +// cleanupQueueAfterProcess(promises, setPluginStatus, reporter) /** * Build images on the fly when they are requested by the browser */ -exports.onCreateDevServer = async ({ - app, - emitter, - boundActionCreators, - actions: { setPluginStatus }, - store, -}) => { - // no need to do set things up when people opt out - if (!normalizedOptions.lazyImageGeneration) { - return - } - - emitter.on(`QUERY_QUEUE_DRAINED`, () => - saveQueueToCache(store, setPluginStatus, jobQueue) - ) - - app.use(async (req, res, next) => { - const queue = getQueueFromCache(store) - if (!queue.has(req.originalUrl)) { - return next() - } - - const job = queue.get(req.originalUrl) - - // wait until the file has been processed and saved to disk - await scheduleJob(job, boundActionCreators, normalizedOptions, false) - // remove job from queue because it has been processed - queue.delete(req.originalUrl) - - saveQueueToCache(store, setPluginStatus, queue) - - return res.sendFile(job.outputPath) - }) -} +// exports.onCreateDevServer = async ({ +// app, +// emitter, +// boundActionCreators, +// actions: { setPluginStatus }, +// store, +// }) => { +// // no need to do set things up when people opt out +// if (!normalizedOptions.lazyImageGeneration) { +// return +// } + +// emitter.on(`QUERY_QUEUE_DRAINED`, () => +// saveQueueToCache(store, setPluginStatus, jobQueue) +// ) + +// app.use(async (req, res, next) => { +// const queue = getQueueFromCache(store) +// if (!queue.has(req.originalUrl)) { +// return next() +// } + +// const job = queue.get(req.originalUrl) + +// // wait until the file has been processed and saved to disk +// await scheduleJob(job, boundActionCreators, normalizedOptions, false) +// // remove job from queue because it has been processed +// queue.delete(req.originalUrl) + +// saveQueueToCache(store, setPluginStatus, queue) + +// return res.sendFile(job.outputPath) +// }) +// } // TODO // exports.formatJobMessage = jobs => { From f56391340f3929adf6f184670649ef6806e51c53 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Wed, 27 Feb 2019 14:55:29 +0100 Subject: [PATCH 35/36] fix tests --- .../__tests__/lazy-image-build/develop.js | 2 +- .../__tests__/lazy-image-build/index.js | 87 ------------------- .../gatsby-pipeline/gatsby-browser.js | 7 -- .../gatsby-pipeline/gatsby-config-default.js | 19 ---- .../gatsby-pipeline/gatsby-config-nolazy.js | 24 ----- 5 files changed, 1 insertion(+), 138 deletions(-) delete mode 100644 integration-tests/gatsby-pipeline/__tests__/lazy-image-build/index.js delete mode 100644 integration-tests/gatsby-pipeline/gatsby-browser.js delete mode 100644 integration-tests/gatsby-pipeline/gatsby-config-default.js delete mode 100644 integration-tests/gatsby-pipeline/gatsby-config-nolazy.js diff --git a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js index f0f4f8eea47b6..9ab20ed639d47 100644 --- a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js +++ b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js @@ -24,7 +24,7 @@ describe(`Lazy images`, () => { const { kill } = await createDevServer() const response = await request( - `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a2541/gatsby-astronaut.png`, + `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/e4795/gatsby-astronaut.png`, { resolveWithFullResponse: true, } diff --git a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/index.js b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/index.js deleted file mode 100644 index 36388781f6c5a..0000000000000 --- a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/index.js +++ /dev/null @@ -1,87 +0,0 @@ -const execa = require(`execa`) -const path = require(`path`) -const fs = require(`fs-extra`) -const glob = require(`glob`) -const request = require(`request-promise-native`) -const createDevServer = require(`../../utils/create-devserver`) -const basePath = path.resolve(__dirname, `../../`) - -// 2 min -jest.setTimeout(2000 * 60) - -const cleanDirs = () => - Promise.all([ - fs.emptyDir(`${basePath}/public`), - fs.emptyDir(`${basePath}/.cache`), - ]) - -const configFiles = { - gatsby: path.join(__dirname, `../../gatsby-config.js`), - default: path.join(__dirname, `../../gatsby-config-default.js`), - nolazy: path.join(__dirname, `../../gatsby-config-nolazy.js`), -} - -describe(`Lazy images`, () => { - beforeAll(async () => { - await cleanDirs() - }) - - beforeEach(async () => { - await fs.copy(configFiles.default, configFiles.gatsby) - }) - - test(`should process images on demand`, async () => { - const { kill } = await createDevServer() - - const response = await request( - `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/cc8d4/gatsby-astronaut.png`, - { - resolveWithFullResponse: true, - } - ) - - await kill() - - expect(response.statusCode).toBe(200) - expect(response.headers[`content-type`]).toBe(`image/png`) - - const images = glob.sync(`${basePath}/public/**/*.png`) - expect(images.length).toBe(1) - }) - - test(`should process the rest of images on build`, async () => { - let images = glob.sync(`${basePath}/public/**/*.png`) - expect(images.length).toBe(1) - - await execa(`yarn`, [`build`], { - cwd: basePath, - env: { NODE_ENV: `production` }, - }) - - images = glob.sync(`${basePath}/public/**/*.png`) - expect(images.length).toBe(6) - }) - - test(`should process all images from a clean build`, async () => { - await cleanDirs() - await execa(`yarn`, [`build`], { - cwd: basePath, - env: { NODE_ENV: `production` }, - }) - - const images = glob.sync(`${basePath}/public/**/*.png`) - expect(images.length).toBe(6) - }) - - test(`should process all images on develop when lazyImageGeneration is false`, async () => { - await fs.copy(configFiles.nolazy, configFiles.gatsby) - - await cleanDirs() - const { kill } = await createDevServer() - - await kill() - - const images = glob.sync(`${basePath}/public/**/*.png`) - expect(images.length).toBe(6) - }) -}) diff --git a/integration-tests/gatsby-pipeline/gatsby-browser.js b/integration-tests/gatsby-pipeline/gatsby-browser.js deleted file mode 100644 index b1e5c316b7f94..0000000000000 --- a/integration-tests/gatsby-pipeline/gatsby-browser.js +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Implement Gatsby's Browser APIs in this file. - * - * See: https://www.gatsbyjs.org/docs/browser-apis/ - */ - -// You can delete this file if you're not using it diff --git a/integration-tests/gatsby-pipeline/gatsby-config-default.js b/integration-tests/gatsby-pipeline/gatsby-config-default.js deleted file mode 100644 index bfda48c3cf314..0000000000000 --- a/integration-tests/gatsby-pipeline/gatsby-config-default.js +++ /dev/null @@ -1,19 +0,0 @@ -module.exports = { - siteMetadata: { - title: `Gatsby Default Starter`, - description: `Kick off your next, great Gatsby project with this default starter. This barebones starter ships with the main Gatsby configuration files you might need.`, - author: `@gatsbyjs`, - }, - plugins: [ - `gatsby-plugin-react-helmet`, - { - resolve: `gatsby-source-filesystem`, - options: { - name: `images`, - path: `${__dirname}/src/images`, - }, - }, - `gatsby-transformer-sharp`, - `gatsby-plugin-sharp`, - ], -} diff --git a/integration-tests/gatsby-pipeline/gatsby-config-nolazy.js b/integration-tests/gatsby-pipeline/gatsby-config-nolazy.js deleted file mode 100644 index 5baca117a8f67..0000000000000 --- a/integration-tests/gatsby-pipeline/gatsby-config-nolazy.js +++ /dev/null @@ -1,24 +0,0 @@ -module.exports = { - siteMetadata: { - title: `Gatsby Default Starter`, - description: `Kick off your next, great Gatsby project with this default starter. This barebones starter ships with the main Gatsby configuration files you might need.`, - author: `@gatsbyjs`, - }, - plugins: [ - `gatsby-plugin-react-helmet`, - { - resolve: `gatsby-source-filesystem`, - options: { - name: `images`, - path: `${__dirname}/src/images`, - }, - }, - `gatsby-transformer-sharp`, - { - resolve: `gatsby-plugin-sharp`, - options: { - lazyImageGeneration: false, - }, - }, - ], -} From 0a614f70b36a537009c9fd5e46c9536cba165c2d Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 6 Mar 2019 21:26:09 +0100 Subject: [PATCH 36/36] schedule jobs immediately (for now) Signed-off-by: Ward Peeters --- .../src/__tests__/index.js | 6 +- .../gatsby-plugin-sharp/src/gatsby-node.js | 65 ++++++++++--------- packages/gatsby-plugin-sharp/src/index.js | 28 +++++--- 3 files changed, 56 insertions(+), 43 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/__tests__/index.js b/packages/gatsby-plugin-sharp/src/__tests__/index.js index ef424e5d7f65d..d930eb987280c 100644 --- a/packages/gatsby-plugin-sharp/src/__tests__/index.js +++ b/packages/gatsby-plugin-sharp/src/__tests__/index.js @@ -18,6 +18,7 @@ const { getImageSize, } = require(`../`) const { scheduleJob } = require(`../scheduler`) +scheduleJob.mockResolvedValue(Promise.resolve()) describe(`gatsby-plugin-sharp`, () => { const args = { @@ -71,8 +72,9 @@ describe(`gatsby-plugin-sharp`, () => { expect(queueResultName.match(/[!@#$^&," ]/)).not.toBe(true) }) - it(`should process immediately when asked`, async () => { - scheduleJob.mockResolvedValue(Promise.resolve()) + // re-enable when image processing on demand is implemented + it.skip(`should process immediately when asked`, async () => { + scheduleJob.mockClear() const result = queueImageResizing({ file: getFileObject(path.join(__dirname, `images/144-density.png`)), args: { width: 3 }, diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js index d92fc977bc91e..8e2abdfd08145 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-node.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-node.js @@ -1,11 +1,11 @@ const { setBoundActionCreators, setPluginOptions, - queue: jobQueue, + // queue: jobQueue, // reportError, } = require(`./index`) -const { scheduleJob } = require(`./scheduler`) -let normalizedOptions = {} +// const { scheduleJob } = require(`./scheduler`) +// let normalizedOptions = {} // const getQueueFromCache = store => { // const pluginStatus = store.getState().status.plugins[`gatsby-plugin-sharp`] @@ -51,37 +51,38 @@ let normalizedOptions = {} exports.onPreBootstrap = ({ actions }, pluginOptions) => { setBoundActionCreators(actions) - normalizedOptions = setPluginOptions(pluginOptions) + setPluginOptions(pluginOptions) + // normalizedOptions = setPluginOptions(pluginOptions) } -/** - * save queue to the cache or process queue - */ -exports.onPostBootstrap = ({ - // store, - boundActionCreators, - // actions: { setPluginStatus }, - // reporter, -}) => { - const promises = [] - for (const [, job] of jobQueue) { - promises.push(scheduleJob(job, boundActionCreators, normalizedOptions)) - } - - return promises - // // Save queue - // saveQueueToCache(store, setPluginStatus, jobQueue) - - // if (normalizedOptions.lazyImageGeneration) { - // return - // } - - // const imageJobs = processQueue(store, boundActionCreators, normalizedOptions) - - // cleanupQueueAfterProcess(imageJobs, setPluginStatus, reporter) - - // return -} +// /** +// * save queue to the cache or process queue +// */ +// exports.onPostBootstrap = ({ +// // store, +// boundActionCreators, +// // actions: { setPluginStatus }, +// // reporter, +// }) => { +// const promises = [] +// for (const [, job] of jobQueue) { +// promises.push(scheduleJob(job, boundActionCreators, normalizedOptions)) +// } + +// return promises +// // // Save queue +// // saveQueueToCache(store, setPluginStatus, jobQueue) + +// // if (normalizedOptions.lazyImageGeneration) { +// // return +// // } + +// // const imageJobs = processQueue(store, boundActionCreators, normalizedOptions) + +// // cleanupQueueAfterProcess(imageJobs, setPluginStatus, reporter) + +// // return +// } /** * Execute all unprocessed images on gatsby build diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index e7c5bf5a14662..fc31465c4c8e6 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -206,21 +206,31 @@ function queueImageResizing({ file, args = {}, reporter }) { queue.set(prefixedSrc, job) + // schedule job immediately - this will be changed when image processing on demand is implemented + const finishedPromise = scheduleJob( + job, + boundActionCreators, + pluginOptions + ).then(() => { + queue.delete(prefixedSrc) + }) + return { src: prefixedSrc, absolutePath: filePath, width, height, aspectRatio, - // finishedPromise is needed to not break our API (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-transformer-sqip/src/extend-node-type.js#L115) - finishedPromise: { - then: (resolve, reject) => { - scheduleJob(job, boundActionCreators, pluginOptions).then(() => { - queue.delete(prefixedSrc) - resolve() - }, reject) - }, - }, + finishedPromise, + // // finishedPromise is needed to not break our API (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-transformer-sqip/src/extend-node-type.js#L115) + // finishedPromise: { + // then: (resolve, reject) => { + // scheduleJob(job, boundActionCreators, pluginOptions).then(() => { + // queue.delete(prefixedSrc) + // resolve() + // }, reject) + // }, + // }, originalName: originalName, } }