refactor(gatsby-plugin-sharp): split single file into more maintainable chunks

.circleci/config.yml

@@ -212,9 +212,9 @@ workflows:
           requires:
             - bootstrap
       - integration_tests_long_term_caching:
-          <<: *ignore_docs
+          <<: *e2e-test-workflow
       - integration_tests_gatsby_pipeline:
-          <<: *ignore_docs
+          <<: *e2e-test-workflow
       - e2e_tests_gatsbygram:
           <<: *e2e-test-workflow
       - e2e_tests_path-prefix:

integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js

@@ -24,7 +24,7 @@ describe(`Lazy images`, () => {
     const { kill } = await createDevServer()
 
     const response = await request(
-      `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a2541/gatsby-astronaut.png`,
+      `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a484e/gatsby-astronaut.png`,
       {
         resolveWithFullResponse: true,
       }
@@ -33,18 +33,22 @@ describe(`Lazy images`, () => {
     await kill()
 
     expect(response.statusCode).toBe(200)
+    expect(response.headers[`content-type`]).toBe(`image/png`)
 
     const images = glob.sync(`${basePath}/public/**/*.png`)
-    expect(images.length).toBe(6)
+    expect(images.length).toBe(1)
   })
 
   test(`should process the rest of images on build`, async () => {
+    let images = glob.sync(`${basePath}/public/**/*.png`)
+    expect(images.length).toBe(1)
+
     await execa(`yarn`, [`build`], {
       cwd: basePath,
       env: { NODE_ENV: `production` },
     })
 
-    const images = glob.sync(`${basePath}/public/**/*.png`)
+    images = glob.sync(`${basePath}/public/**/*.png`)
     expect(images.length).toBe(6)
   })
 

jest.config.js

@@ -32,6 +32,7 @@ module.exports = {
   moduleNameMapper: {
     "^highlight.js$": `<rootDir>/node_modules/highlight.js/lib/index.js`,
   },
+  snapshotSerializers: [`jest-serializer-path`],
   collectCoverage: useCoverage,
   coverageReporters: [`json-summary`, `text`, `html`, `cobertura`],
   coverageThreshold: {

package.json

@@ -24,6 +24,7 @@
     "jest": "^24.0.0",
     "jest-cli": "^24.0.0",
     "jest-junit": "^6.1.0",
+    "jest-serializer-path": "^0.1.15",
     "lerna": "^3.10.7",
     "lint-staged": "^8.0.4",
     "markdown-magic": "^0.1.25",

packages/gatsby-plugin-sharp/.gitignore

@@ -1,4 +1,2 @@
-/index.js
 /node_modules
-/gatsby-node.js
-/duotone.js
+/*.js

packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap

@@ -15,14 +15,14 @@ Object {
   "aspectRatio": 2.0661764705882355,
   "base64": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAABYlAAAWJQFJUiTwAAABAklEQVQoz61R7WrCQBDM+7+Cv1qhlIogoUWMoH8KrfgOLVXRKDV4d7kkl6/pbsxpqBIo9mBuZ7N3k51bB7TKsrwZVsexyS2rKerwFhYFVI1DnkMQOHIuCaLB+ayuwXlMaApXgvMoxjTU8MIQE4rMx8SnNffUkU/qM/bbiPAeRfgw5n8tB+QgO1kmUvwROdm0kYHmUNoe+NoU2wZTdVjQH7Isg9YRpFIXor8vpGmKjb/FQQgslit8fi3wvQ/OHXLhxRvDHY7Qd5+x3e2udmNzTUN4fZvhoddH566L+8cn9AZuVcvpLR3e9kEAISWEkDAmbRXkGCcJ1r4PRRNPEkPu9Kn2Azs0CqJYU3JKAAAAAElFTkSuQmCC",
   "density": 144,
-  "originalImg": "/static/1234/a3030/test.png",
+  "originalImg": "/static/1234/29cdc/test.png",
   "originalName": undefined,
   "presentationHeight": 68,
   "presentationWidth": 141,
   "sizes": "(max-width: 141px) 100vw, 141px",
-  "src": "/static/1234/a3030/test.png",
-  "srcSet": "/static/1234/56b42/test.png 200w,
-/static/1234/a3030/test.png 281w",
+  "src": "/static/1234/29cdc/test.png",
+  "srcSet": "/static/1234/223da/test.png 200w,
+/static/1234/29cdc/test.png 281w",
   "srcSetType": "image/png",
 }
 `;
@@ -32,14 +32,14 @@ Object {
   "aspectRatio": 2.0661764705882355,
   "base64": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAABYlAAAWJQFJUiTwAAABAklEQVQoz61R7WrCQBDM+7+Cv1qhlIogoUWMoH8KrfgOLVXRKDV4d7kkl6/pbsxpqBIo9mBuZ7N3k51bB7TKsrwZVsexyS2rKerwFhYFVI1DnkMQOHIuCaLB+ayuwXlMaApXgvMoxjTU8MIQE4rMx8SnNffUkU/qM/bbiPAeRfgw5n8tB+QgO1kmUvwROdm0kYHmUNoe+NoU2wZTdVjQH7Isg9YRpFIXor8vpGmKjb/FQQgslit8fi3wvQ/OHXLhxRvDHY7Qd5+x3e2udmNzTUN4fZvhoddH566L+8cn9AZuVcvpLR3e9kEAISWEkDAmbRXkGCcJ1r4PRRNPEkPu9Kn2Azs0CqJYU3JKAAAAAElFTkSuQmCC",
   "density": 144,
-  "originalImg": "/static/1234/67b29/test.png",
+  "originalImg": "/static/1234/55fae/test.png",
   "originalName": undefined,
   "presentationHeight": 136,
   "presentationWidth": 281,
   "sizes": "(max-width: 281px) 100vw, 281px",
-  "src": "/static/1234/67b29/test.png",
-  "srcSet": "/static/1234/4df82/test.png 200w,
-/static/1234/67b29/test.png 281w",
+  "src": "/static/1234/55fae/test.png",
+  "srcSet": "/static/1234/6d94b/test.png 200w,
+/static/1234/55fae/test.png 281w",
   "srcSetType": "image/png",
 }
 `;
@@ -49,13 +49,13 @@ Object {
   "aspectRatio": 1,
   "base64": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAIAAAAC64paAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAGElEQVQ4y2P4TwFgGNU8qnlU86jmgdUMAOBEq42KgAyMAAAAAElFTkSuQmCC",
   "density": 72,
-  "originalImg": "/static/1234/4ce1a/test.png",
+  "originalImg": "/static/1234/7bd23/test.png",
   "originalName": undefined,
   "presentationHeight": 1,
   "presentationWidth": 1,
   "sizes": "(max-width: 1px) 100vw, 1px",
-  "src": "/static/1234/4ce1a/test.png",
-  "srcSet": "/static/1234/4ce1a/test.png 1w",
+  "src": "/static/1234/7bd23/test.png",
+  "srcSet": "/static/1234/7bd23/test.png 1w",
   "srcSetType": "image/png",
 }
 `;
@@ -70,3 +70,66 @@ Object {
   "width": 746,
 }
 `;
+
+exports[`gatsby-plugin-sharp queueImageResizing should process immediately when asked 1`] = `
+[MockFunction] {
+  "calls": Array [
+    Array [
+      Object {
+        "args": Object {
+          "base64": false,
+          "duotone": false,
+          "grayscale": false,
+          "jpegProgressive": true,
+          "maxWidth": 800,
+          "pathPrefix": "",
+          "pngCompressionLevel": 9,
+          "pngCompressionSpeed": 4,
+          "quality": 50,
+          "sizeByPixelDensity": false,
+          "toFormat": "png",
+          "width": 3,
+        },
+        "inputPath": "<PROJECT_ROOT>/packages/gatsby-plugin-sharp/src/__tests__/images/144-density.png",
+        "outputPath": "<PROJECT_ROOT>/public/static/1234/39ca0/test.png",
+        "src": "/static/1234/39ca0/test.png",
+      },
+      Object {
+        "addThirdPartySchema": [Function],
+        "createJob": [Function],
+        "createNode": [Function],
+        "createNodeField": [Function],
+        "createPage": [Function],
+        "createPageDependency": [Function],
+        "createParentChildLink": [Function],
+        "createRedirect": [Function],
+        "deleteComponentsDependencies": [Function],
+        "deleteNode": [Function],
+        "deleteNodes": [Function],
+        "deletePage": [Function],
+        "endJob": [Function],
+        "replaceComponentQuery": [Function],
+        "replaceStaticQuery": [Function],
+        "replaceWebpackConfig": [Function],
+        "setBabelOptions": [Function],
+        "setBabelPlugin": [Function],
+        "setBabelPreset": [Function],
+        "setJob": [Function],
+        "setPluginStatus": [Function],
+        "setWebpackConfig": [Function],
+        "touchNode": [Function],
+      },
+      Object {
+        "stripMetadata": true,
+        "useMozJpeg": false,
+      },
+    ],
+  ],
+  "results": Array [
+    Object {
+      "type": "return",
+      "value": Promise {},
+    },
+  ],
+}
+`;

packages/gatsby-plugin-sharp/src/__tests__/index.js

@@ -1,5 +1,6 @@
 const path = require(`path`)
 const fs = require(`fs-extra`)
+jest.mock(`../scheduler`)
 
 jest.mock(`async/queue`, () => () => {
   return {
@@ -16,6 +17,7 @@ const {
   queueImageResizing,
   getImageSize,
 } = require(`../`)
+const { scheduleJob } = require(`../scheduler`)
 
 describe(`gatsby-plugin-sharp`, () => {
   const args = {
@@ -50,7 +52,7 @@ describe(`gatsby-plugin-sharp`, () => {
       // test name encoding with various characters
       const testName = `spaces and '"@#$%^&,`
 
-      const queueResult = await queueImageResizing({
+      const queueResult = queueImageResizing({
         file: getFileObject(
           path.join(__dirname, `images/144-density.png`),
           testName
@@ -68,6 +70,18 @@ describe(`gatsby-plugin-sharp`, () => {
       expect(testName.match(/[!@#$^&," ]/)).not.toBe(false)
       expect(queueResultName.match(/[!@#$^&," ]/)).not.toBe(true)
     })
+
+    it(`should process immediately when asked`, async () => {
+      scheduleJob.mockResolvedValue(Promise.resolve())
+      const result = queueImageResizing({
+        file: getFileObject(path.join(__dirname, `images/144-density.png`)),
+        args: { width: 3 },
+      })
+
+      await result.finishedPromise
+
+      expect(scheduleJob).toMatchSnapshot()
+    })
   })
 
   describe(`fluid`, () => {

packages/gatsby-plugin-sharp/src/gatsby-node.js

@@ -1,10 +1,96 @@
-const { setBoundActionCreators, setPluginOptions } = require(`./index`)
+const {
+  setBoundActionCreators,
+  setPluginOptions,
+  queue: jobQueue,
+  reportError,
+} = require(`./index`)
+const { scheduleJob } = require(`./scheduler`)
 
-exports.onPreInit = ({ actions }, pluginOptions) => {
+const getQueueFromCache = async cache => {
+  const rawQueue = await cache.get(`queue`)
+
+  if (!rawQueue) {
+    return new Map()
+  }
+
+  return new Map(rawQueue)
+}
+
+const saveQueueToCache = async (cache, queue) => {
+  const cachedQueue = await getQueueFromCache(cache)
+
+  // merge both queues
+  for (const [key, job] of cachedQueue) {
+    if (!queue.has(key)) {
+      queue.set(key, job)
+    }
+  }
+
+  // JSON.stringify doesn't work on an Map so we need to convert it to an array
+  return cache.set(`queue`, Array.from(queue))
+}
+
+exports.onPreBootstrap = ({ actions }, pluginOptions) => {
   setBoundActionCreators(actions)
   setPluginOptions(pluginOptions)
 }
 
+/**
+ * save queue to the cache
+ */
+exports.onPostBootstrap = async ({ cache, store }) =>
+  saveQueueToCache(cache, jobQueue)
+
+/**
+ * Execute all unprocessed images on gatsby build
+ */
+let promises = []
+exports.onPreBuild = async ({ cache, boundActionCreators }, pluginOptions) => {
+  const cachedQueue = await getQueueFromCache(cache)
+
+  for (const [, job] of cachedQueue) {
+    promises.push(scheduleJob(job, boundActionCreators, pluginOptions))
+  }
+}
+
+/**
+ * wait for all images to be processed
+ */
+exports.onPostBuild = ({ cache, reporter }) =>
+  Promise.all(promises)
+    .then(() => cache.set(`queue`, []))
+    .catch(({ err, message }) => {
+      reportError(message || err.message, err, reporter)
+    })
+
+/**
+ * Build images on the fly when they are requested by the browser
+ */
+exports.onCreateDevServer = async (
+  { app, cache, emitter, boundActionCreators },
+  pluginOptions
+) => {
+  emitter.on(`QUERY_QUEUE_DRAINED`, () => saveQueueToCache(cache, jobQueue))
+
+  app.use(async (req, res, next) => {
+    const queue = await getQueueFromCache(cache)
+    if (!queue.has(req.originalUrl)) {
+      return next()
+    }
+
+    const job = queue.get(req.originalUrl)
+
+    // wait until the file has been processed and saved to disk
+    await scheduleJob(job, boundActionCreators, pluginOptions, false)
+    // remove job from queue because it has been processed
+    queue.delete(req.originalUrl)
+
+    await saveQueueToCache(cache, queue)
+
+    return res.sendFile(job.outputPath)
+  })
+}
+
 // TODO
 // exports.formatJobMessage = jobs => {
 // return {