From 98e680b6ca3adeb719b1f17084a84e3afa293fb5 Mon Sep 17 00:00:00 2001 From: Anthony Marcar Date: Tue, 16 Apr 2019 06:07:58 -0700 Subject: [PATCH 01/16] feat(gatsby): Page data without compilation hash (#13139) **Note: merges to `per-page-manifest`, not master. See https://github.com/gatsbyjs/gatsby/pull/13004 for more info** ## Description This PR saves the `page-data.json` during query running. In order to minimize the size of PRs, my strategy is to save the page-data.json along with the normal query results, and then gradually shift functionality over to use `page-data.json` instead of `data.json`. Once all those PRs are merged, we'll be able to go back and delete the static query results, jsonNames, dataPaths, data.json etc. It does mean that we'll be storing double the amount of query results on disk. Hopefully that's ok in the interim. Compilation-hash will be added in future PRs. ## Related Issues - Sub-PR of https://github.com/gatsbyjs/gatsby/pull/13004 --- packages/gatsby/src/query/query-runner.js | 31 ++++++++++++++++------- packages/gatsby/src/utils/page-data.js | 21 +++++++++++++++ 2 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 packages/gatsby/src/utils/page-data.js diff --git a/packages/gatsby/src/query/query-runner.js b/packages/gatsby/src/query/query-runner.js index 6592bce01f065..0a4954a995787 100644 --- a/packages/gatsby/src/query/query-runner.js +++ b/packages/gatsby/src/query/query-runner.js @@ -11,6 +11,7 @@ const { generatePathChunkName } = require(`../utils/js-chunk-names`) const { formatErrorDetails } = require(`./utils`) const mod = require(`hash-mod`)(999) const { boundActionCreators } = require(`../redux/actions`) +const pageDataUtil = require(`../utils/page-data`) const resultHashes = {} @@ -91,20 +92,23 @@ ${formatErrorDetails(errorDetails)}`) .createHash(`sha1`) .update(resultJSON) .digest(`base64`) + + if (resultHashes[queryJob.id] !== resultHash) { + resultHashes[queryJob.id] = resultHash + // Remove potentially unsafe characters. This increases chances of collisions // slightly but it should still be very safe + we get a shorter // url vs hex. - .replace(/[^a-zA-Z0-9-_]/g, ``) + const readableResultHash = resultHash.replace(/[^a-zA-Z0-9-_]/g, ``) - let dataPath - if (queryJob.isPage) { - dataPath = `${generatePathChunkName(queryJob.jsonName)}-${resultHash}` - } else { - dataPath = queryJob.hash - } + let dataPath + if (queryJob.isPage) { + const pathChunkName = generatePathChunkName(queryJob.jsonName) + dataPath = `${pathChunkName}-${readableResultHash}` + } else { + dataPath = queryJob.hash + } - if (resultHashes[queryJob.id] !== resultHash) { - resultHashes[queryJob.id] = resultHash let modInt = `` // We leave StaticQuery results at public/static/d // as the babel plugin has that path hard-coded @@ -136,6 +140,15 @@ ${formatErrorDetails(errorDetails)}`) value: dataPath, }, }) + + // Save page-data.json. This isn't used yet but is part of + // https://github.com/gatsbyjs/gatsby/pull/13004 + if (queryJob.isPage) { + const publicDir = path.join(program.directory, `public`) + const { pages } = store.getState() + const page = pages.get(queryJob.id) + await pageDataUtil.write({ publicDir }, page, result) + } } boundActionCreators.pageQueryRun({ diff --git a/packages/gatsby/src/utils/page-data.js b/packages/gatsby/src/utils/page-data.js new file mode 100644 index 0000000000000..7583664a769b6 --- /dev/null +++ b/packages/gatsby/src/utils/page-data.js @@ -0,0 +1,21 @@ +const fs = require(`fs-extra`) +const path = require(`path`) + +const getFilePath = ({ publicDir }, pagePath) => { + const fixedPagePath = pagePath === `/` ? `index` : pagePath + return path.join(publicDir, `page-data`, fixedPagePath, `page-data.json`) +} + +const write = async ({ publicDir }, page, result) => { + const filePath = getFilePath({ publicDir }, page.path) + const body = { + componentChunkName: page.componentChunkName, + path: page.path, + result, + } + await fs.outputFile(filePath, JSON.stringify(body)) +} + +module.exports = { + write, +} From 9bf7ad3d98595de09f56b741144285e5914570bb Mon Sep 17 00:00:00 2001 From: Anthony Marcar Date: Tue, 16 Apr 2019 18:05:28 -0700 Subject: [PATCH 02/16] Websocket manager use page data (#13389) * add utils/page-data.read * read websocket page data from utils/page-data --- packages/gatsby/src/utils/page-data.js | 7 +++++ .../gatsby/src/utils/websocket-manager.js | 28 +++++++++---------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/packages/gatsby/src/utils/page-data.js b/packages/gatsby/src/utils/page-data.js index 7583664a769b6..276ec3a6723aa 100644 --- a/packages/gatsby/src/utils/page-data.js +++ b/packages/gatsby/src/utils/page-data.js @@ -6,6 +6,12 @@ const getFilePath = ({ publicDir }, pagePath) => { return path.join(publicDir, `page-data`, fixedPagePath, `page-data.json`) } +const read = async ({ publicDir }, pagePath) => { + const filePath = getFilePath({ publicDir }, pagePath) + const rawPageData = await fs.readFile(filePath, `utf-8`) + return JSON.parse(rawPageData) +} + const write = async ({ publicDir }, page, result) => { const filePath = getFilePath({ publicDir }, page.path) const body = { @@ -17,5 +23,6 @@ const write = async ({ publicDir }, page, result) => { } module.exports = { + read, write, } diff --git a/packages/gatsby/src/utils/websocket-manager.js b/packages/gatsby/src/utils/websocket-manager.js index 93812e795d6fb..62d858d12bcc3 100644 --- a/packages/gatsby/src/utils/websocket-manager.js +++ b/packages/gatsby/src/utils/websocket-manager.js @@ -3,6 +3,7 @@ const path = require(`path`) const { store } = require(`../redux`) const fs = require(`fs`) +const pageDataUtil = require(`../utils/page-data`) type QueryResult = { id: string, @@ -32,27 +33,24 @@ const readCachedResults = (dataFileName: string, directory: string): object => { * @param {string} pagePath Path to a page. * @param {string} directory Root directory of current project. */ -const getCachedPageData = ( +const getCachedPageData = async ( pagePath: string, directory: string ): QueryResult => { - const { jsonDataPaths, pages } = store.getState() - const page = pages.get(pagePath) - if (!page) { - return null - } - const dataPath = jsonDataPaths[page.jsonName] - if (typeof dataPath === `undefined`) { + const { program } = store.getState() + const publicDir = path.join(program.directory, `public`) + try { + const pageData = await pageDataUtil.read({ publicDir }, pagePath) + return { + result: pageData.result, + id: pagePath, + } + } catch (err) { console.log( `Error loading a result for the page query in "${pagePath}". Query was not run and no cached result was found.` ) return undefined } - - return { - result: readCachedResults(dataPath, directory), - id: pagePath, - } } /** @@ -156,9 +154,9 @@ class WebsocketManager { } } - const getDataForPath = path => { + const getDataForPath = async path => { if (!this.pageResults.has(path)) { - const result = getCachedPageData(path, this.programDir) + const result = await getCachedPageData(path, this.programDir) if (result) { this.pageResults.set(path, result) } else { From c7def019a322c530bf2974bd4fe57158af4cd26f Mon Sep 17 00:00:00 2001 From: Anthony Marcar Date: Mon, 29 Apr 2019 16:22:30 -0700 Subject: [PATCH 03/16] Loader use page data (#13409) * page-data loader working for production-app * get new loader.getPage stuff working with gatsby develop * fix static-entry.js test * remove loadPageDataSync (will be in next PR) * use array of matchPaths * Deprecate various loader methods * remove console.log * document why fetchPageData needs to check that response is JSON * in offline, use prefetchedResources.push(...resourceUrls) * root.js remove else block * loader.js make* -> create* * loader drop else block * pass correct path/resourceUrls to onPostPrefetch * s/err => null/() => null/ * Extract loadComponent from loadPage * remove pageData from window object * update jest snapshots for static-entry (to use window.pagePath) * add loadPageOr404 * preload 404 page in background * normalize page paths * comment out resource-loading-resilience.js (will fix later) --- .../resource-loading-resilience.js | 210 +++---- .../src/gatsby-browser.js | 7 +- .../src/gatsby-browser.js | 22 +- .../__snapshots__/static-entry.js.snap | 6 +- .../gatsby/cache-dir/__tests__/find-page.js | 67 -- .../cache-dir/__tests__/static-entry.js | 28 +- .../gatsby/cache-dir/api-runner-browser.js | 7 + packages/gatsby/cache-dir/app.js | 11 +- packages/gatsby/cache-dir/ensure-resources.js | 10 +- packages/gatsby/cache-dir/find-page.js | 55 -- packages/gatsby/cache-dir/json-store.js | 7 +- packages/gatsby/cache-dir/loader.js | 588 ++++++++---------- packages/gatsby/cache-dir/navigation.js | 4 +- .../gatsby/cache-dir/normalize-page-path.js | 12 + packages/gatsby/cache-dir/production-app.js | 25 +- .../cache-dir/public-page-renderer-dev.js | 2 +- .../cache-dir/public-page-renderer-prod.js | 3 +- packages/gatsby/cache-dir/root.js | 44 +- packages/gatsby/cache-dir/socketIo.js | 23 +- packages/gatsby/cache-dir/static-entry.js | 64 +- packages/gatsby/src/query/pages-writer.js | 2 +- packages/gatsby/src/utils/api-browser-docs.js | 8 +- 22 files changed, 505 insertions(+), 700 deletions(-) delete mode 100644 packages/gatsby/cache-dir/__tests__/find-page.js delete mode 100644 packages/gatsby/cache-dir/find-page.js create mode 100644 packages/gatsby/cache-dir/normalize-page-path.js diff --git a/e2e-tests/production-runtime/cypress/integration/resource-loading-resilience.js b/e2e-tests/production-runtime/cypress/integration/resource-loading-resilience.js index 2e1093969b355..b4c42c81c58da 100644 --- a/e2e-tests/production-runtime/cypress/integration/resource-loading-resilience.js +++ b/e2e-tests/production-runtime/cypress/integration/resource-loading-resilience.js @@ -1,116 +1,118 @@ -Cypress.on(`uncaught:exception`, (err, runnable) => { - // returning false here prevents Cypress from - // failing the test - console.log(err) - return false -}) +// TODO need to update to work with page-data.json -const waitForAPIOptions = { - timeout: 3000, -} +// Cypress.on(`uncaught:exception`, (err, runnable) => { +// // returning false here prevents Cypress from +// // failing the test +// console.log(err) +// return false +// }) -const runTests = () => { - it(`Loads index`, () => { - cy.visit(`/`).waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) - cy.getTestElement(`dom-marker`).contains(`index`) - }) +// const waitForAPIOptions = { +// timeout: 3000, +// } - it(`Navigates to second page`, () => { - cy.getTestElement(`page2`).click() - cy.waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) - .location(`pathname`) - .should(`equal`, `/page-2/`) - cy.getTestElement(`dom-marker`).contains(`page-2`) - }) +// const runTests = () => { +// it(`Loads index`, () => { +// cy.visit(`/`).waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) +// cy.getTestElement(`dom-marker`).contains(`index`) +// }) - it(`Navigates to 404 page`, () => { - cy.getTestElement(`404`).click() - cy.waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) - .location(`pathname`) - .should(`equal`, `/page-3/`) - cy.getTestElement(`dom-marker`).contains(`404`) - }) +// it(`Navigates to second page`, () => { +// cy.getTestElement(`page2`).click() +// cy.waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) +// .location(`pathname`) +// .should(`equal`, `/page-2/`) +// cy.getTestElement(`dom-marker`).contains(`page-2`) +// }) - it(`Loads 404`, () => { - cy.visit(`/page-3/`, { - failOnStatusCode: false, - }).waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) - cy.getTestElement(`dom-marker`).contains(`404`) - }) +// it(`Navigates to 404 page`, () => { +// cy.getTestElement(`404`).click() +// cy.waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) +// .location(`pathname`) +// .should(`equal`, `/page-3/`) +// cy.getTestElement(`dom-marker`).contains(`404`) +// }) - it(`Can navigate from 404 to index`, () => { - cy.getTestElement(`index`).click() - cy.waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) - .location(`pathname`) - .should(`equal`, `/`) - cy.getTestElement(`dom-marker`).contains(`index`) - }) -} +// it(`Loads 404`, () => { +// cy.visit(`/page-3/`, { +// failOnStatusCode: false, +// }).waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) +// cy.getTestElement(`dom-marker`).contains(`404`) +// }) -describe(`Every resources available`, () => { - it(`Restore resources`, () => { - cy.exec(`npm run chunks -- restore`) - }) - runTests() -}) +// it(`Can navigate from 404 to index`, () => { +// cy.getTestElement(`index`).click() +// cy.waitForAPIorTimeout(`onRouteUpdate`, waitForAPIOptions) +// .location(`pathname`) +// .should(`equal`, `/`) +// cy.getTestElement(`dom-marker`).contains(`index`) +// }) +// } -describe(`Missing top level resources`, () => { - describe(`Deleted pages manifest`, () => { - it(`Block resources`, () => { - cy.exec(`npm run chunks -- restore`) - cy.exec(`npm run chunks -- block pages-manifest`) - }) - runTests() - }) +// describe(`Every resources available`, () => { +// it(`Restore resources`, () => { +// cy.exec(`npm run chunks -- restore`) +// }) +// runTests() +// }) - describe(`Deleted app chunk assets`, () => { - it(`Block resources`, () => { - cy.exec(`npm run chunks -- restore`) - cy.exec(`npm run chunks -- block app`) - }) - runTests() - }) -}) +// describe(`Missing top level resources`, () => { +// describe(`Deleted pages manifest`, () => { +// it(`Block resources`, () => { +// cy.exec(`npm run chunks -- restore`) +// cy.exec(`npm run chunks -- block pages-manifest`) +// }) +// runTests() +// }) -const runSuiteForPage = (label, path) => { - describe(`Missing "${label}" resources`, () => { - describe(`Missing "${label}" page query results`, () => { - it(`Block resources`, () => { - cy.exec(`npm run chunks -- restore`) - cy.exec(`npm run chunks -- block-page ${path} query-result`) - }) - runTests() - }) - describe(`Missing "${label}" page page-template asset`, () => { - it(`Block resources`, () => { - cy.exec(`npm run chunks -- restore`) - cy.exec(`npm run chunks -- block-page ${path} page-template`) - }) - runTests() - }) - describe(`Missing "${label}" page extra assets`, () => { - it(`Block resources`, () => { - cy.exec(`npm run chunks -- restore`) - cy.exec(`npm run chunks -- block-page ${path} extra`) - }) - runTests() - }) - describe(`Missing all "${label}" page assets`, () => { - it(`Block resources`, () => { - cy.exec(`npm run chunks -- restore`) - cy.exec(`npm run chunks -- block-page ${path} all`) - }) - runTests() - }) - }) -} +// describe(`Deleted app chunk assets`, () => { +// it(`Block resources`, () => { +// cy.exec(`npm run chunks -- restore`) +// cy.exec(`npm run chunks -- block app`) +// }) +// runTests() +// }) +// }) -runSuiteForPage(`Index`, `/`) -runSuiteForPage(`Page-2`, `/page-2/`) -runSuiteForPage(`404`, `/404.html`) +// const runSuiteForPage = (label, path) => { +// describe(`Missing "${label}" resources`, () => { +// describe(`Missing "${label}" page query results`, () => { +// it(`Block resources`, () => { +// cy.exec(`npm run chunks -- restore`) +// cy.exec(`npm run chunks -- block-page ${path} query-result`) +// }) +// runTests() +// }) +// describe(`Missing "${label}" page page-template asset`, () => { +// it(`Block resources`, () => { +// cy.exec(`npm run chunks -- restore`) +// cy.exec(`npm run chunks -- block-page ${path} page-template`) +// }) +// runTests() +// }) +// describe(`Missing "${label}" page extra assets`, () => { +// it(`Block resources`, () => { +// cy.exec(`npm run chunks -- restore`) +// cy.exec(`npm run chunks -- block-page ${path} extra`) +// }) +// runTests() +// }) +// describe(`Missing all "${label}" page assets`, () => { +// it(`Block resources`, () => { +// cy.exec(`npm run chunks -- restore`) +// cy.exec(`npm run chunks -- block-page ${path} all`) +// }) +// runTests() +// }) +// }) +// } -describe(`Cleanup`, () => { - it(`Restore resources`, () => { - cy.exec(`npm run chunks -- restore`) - }) -}) +// runSuiteForPage(`Index`, `/`) +// runSuiteForPage(`Page-2`, `/page-2/`) +// runSuiteForPage(`404`, `/404.html`) + +// describe(`Cleanup`, () => { +// it(`Restore resources`, () => { +// cy.exec(`npm run chunks -- restore`) +// }) +// }) diff --git a/packages/gatsby-plugin-guess-js/src/gatsby-browser.js b/packages/gatsby-plugin-guess-js/src/gatsby-browser.js index 1e16ce0079505..102c8a164246f 100644 --- a/packages/gatsby-plugin-guess-js/src/gatsby-browser.js +++ b/packages/gatsby-plugin-guess-js/src/gatsby-browser.js @@ -12,7 +12,7 @@ exports.onRouteUpdate = ({ location }) => { initialPath = location.pathname } -exports.onPrefetchPathname = ({ getResourcesForPathname }, pluginOptions) => { +exports.onPrefetchPathname = ({ loadPage }, pluginOptions) => { if (process.env.NODE_ENV !== `production`) return const matchedPaths = Object.keys( @@ -24,6 +24,7 @@ exports.onPrefetchPathname = ({ getResourcesForPathname }, pluginOptions) => { // Don't prefetch from client for the initial path as we did that // during SSR - if (!(notNavigated && initialPath === window.location.pathname)) - matchedPaths.forEach(getResourcesForPathname) + if (!(notNavigated && initialPath === window.location.pathname)) { + matchedPaths.forEach(loadPage) + } } diff --git a/packages/gatsby-plugin-offline/src/gatsby-browser.js b/packages/gatsby-plugin-offline/src/gatsby-browser.js index a9ee410438ff9..5db5f76e8e155 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-browser.js +++ b/packages/gatsby-plugin-offline/src/gatsby-browser.js @@ -1,12 +1,9 @@ exports.registerServiceWorker = () => true -const prefetchedPathnames = [] +const prefetchedResources = [] const whitelistedPathnames = [] -exports.onServiceWorkerActive = ({ - getResourceURLsForPathname, - serviceWorker, -}) => { +exports.onServiceWorkerActive = ({ serviceWorker }) => { // if the SW has just updated then reset whitelisted paths and don't cache // stuff, since we're on the old revision until we navigate to another page if (window.___swUpdated) { @@ -26,15 +23,6 @@ exports.onServiceWorkerActive = ({ .call(nodes) .map(node => node.src || node.href || node.getAttribute(`data-href`)) - // Loop over all resources and fetch the page component and JSON - // to add it to the sw cache. - const prefetchedResources = [] - prefetchedPathnames.forEach(path => - getResourceURLsForPathname(path).forEach(resource => - prefetchedResources.push(resource) - ) - ) - const resources = [...headerResources, ...prefetchedResources] resources.forEach(resource => { // Create a prefetch link for each resource, so Workbox runtime-caches them @@ -69,12 +57,12 @@ function whitelistPathname(pathname, includesPrefix) { } } -exports.onPostPrefetchPathname = ({ pathname }) => { +exports.onPostPrefetch = ({ path, resourceUrls }) => { // do nothing if the SW has just updated, since we still have old pages in // memory which we don't want to be whitelisted if (window.___swUpdated) return - whitelistPathname(pathname, false) + whitelistPathname(path, false) // if SW is not installed, we need to record any prefetches // that happen so we can then add them to SW cache once installed @@ -85,6 +73,6 @@ exports.onPostPrefetchPathname = ({ pathname }) => { navigator.serviceWorker.controller.state === `activated` ) ) { - prefetchedPathnames.push(pathname) + prefetchedResources.push(...resourceUrls) } } diff --git a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap index 2137dd0c59a58..ebd71a1b4df16 100644 --- a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap +++ b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap @@ -6,8 +6,8 @@ exports[`develop-static-entry onPreRenderHTML can be used to replace postBodyCom exports[`develop-static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"
div3
div2
div1
"`; -exports[`static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"
"`; +exports[`static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"
"`; -exports[`static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"
div3
div2
div1
"`; +exports[`static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"
div3
div2
div1
"`; -exports[`static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"
div3
div2
div1
"`; +exports[`static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"
div3
div2
div1
"`; diff --git a/packages/gatsby/cache-dir/__tests__/find-page.js b/packages/gatsby/cache-dir/__tests__/find-page.js deleted file mode 100644 index c998b8e813c9a..0000000000000 --- a/packages/gatsby/cache-dir/__tests__/find-page.js +++ /dev/null @@ -1,67 +0,0 @@ -const pageFinderFactory = require(`../find-page`).default - -let findPage - -describe(`find-page`, () => { - beforeEach(() => { - const newPages = [ - { - path: `/about/`, - componentChunkName: `page-component---src-pages-test-js`, - jsonName: `about.json`, - }, - { - path: `/about/me/`, - componentChunkName: `page-component---src-pages-test-js`, - jsonName: `about-me.json`, - }, - { - path: `/about/the best/`, - componentChunkName: `page-component---src-pages-test-js`, - jsonName: `the-best.json`, - }, - { - path: `/app/`, - matchPath: `/app/*`, - componentChunkName: `page-component---src-pages-app-js`, - jsonName: `app.json`, - }, - ] - findPage = pageFinderFactory(newPages) - }) - - it(`can find a page`, () => { - expect(findPage(`/about/`).path).toBe(`/about/`) - expect(findPage(`/about/me/`).path).toBe(`/about/me/`) - }) - - it(`can find a page with space in its path`, () => { - expect(findPage(`/about/the best/`).path).toBe(`/about/the best/`) - expect(findPage(`/about/the%20best/`).path).toBe(`/about/the best/`) - }) - - it(`can find a client only path`, () => { - expect(findPage(`/about/super-duper/`)).toBeUndefined() - expect(findPage(`/app/client/only/path`).path).toBe(`/app/`) - }) - - it(`can find links with hashes`, () => { - expect(findPage(`/about/me/#hashtagawesome`).path).toBe(`/about/me/`) - }) - - it(`can find links with search query`, () => { - expect(findPage(`/about/me/?query=awesome`).path).toBe(`/about/me/`) - }) - - it(`handles finding prefixed links`, () => { - const newPages = [ - { - path: `/about/`, - componentChunkName: `page-component---src-pages-test-js`, - jsonName: `about.json`, - }, - ] - const findPage2 = pageFinderFactory(newPages, `/my-test-prefix`) - expect(findPage2(`/my-test-prefix/about/`).path).toBe(`/about/`) - }) -}) diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js index 8cb6487dd496b..0f5253ccf1c08 100644 --- a/packages/gatsby/cache-dir/__tests__/static-entry.js +++ b/packages/gatsby/cache-dir/__tests__/static-entry.js @@ -30,32 +30,14 @@ jest.mock( } ) -jest.mock( - `../data.json`, - () => { - return { - dataPaths: [ - { - [`about.json`]: `/400/about`, - }, - ], - pages: [ - { - path: `/about/`, - componentChunkName: `page-component---src-pages-test-js`, - jsonName: `about.json`, - }, - ], - } - }, - { - virtual: true, - } -) - const MOCK_FILE_INFO = { [`${process.cwd()}/public/webpack.stats.json`]: `{}`, [`${process.cwd()}/public/chunk-map.json`]: `{}`, + [`${process.cwd()}/public/page-data/about/page-data.json`]: JSON.stringify({ + componentChunkName: `page-component---src-pages-test-js`, + path: `/about/`, + jsonName: `about.json`, + }), } let StaticEntry diff --git a/packages/gatsby/cache-dir/api-runner-browser.js b/packages/gatsby/cache-dir/api-runner-browser.js index 0a4ff8b1168f4..f956b0494e13f 100644 --- a/packages/gatsby/cache-dir/api-runner-browser.js +++ b/packages/gatsby/cache-dir/api-runner-browser.js @@ -3,6 +3,8 @@ const { getResourcesForPathname, getResourcesForPathnameSync, getResourceURLsForPathname, + loadPage, + getPage, } = require(`./loader`).publicLoader exports.apiRunner = (api, args = {}, defaultReturn, argTransform) => { @@ -22,9 +24,14 @@ exports.apiRunner = (api, args = {}, defaultReturn, argTransform) => { return undefined } + // Deprecated April 2019. Use `getPage` instead args.getResourcesForPathnameSync = getResourcesForPathnameSync + // Deprecated April 2019. Use `loadPage` instead args.getResourcesForPathname = getResourcesForPathname + // Deprecated April 2019. Use resources passed in `onPostPrefetch` instead args.getResourceURLsForPathname = getResourceURLsForPathname + args.loadPage = loadPage + args.getPage = getPage const result = plugin.plugin[api](args, plugin.options) if (result && argTransform) { diff --git a/packages/gatsby/cache-dir/app.js b/packages/gatsby/cache-dir/app.js index 4f502082ec8d9..c09e618b2b48a 100644 --- a/packages/gatsby/cache-dir/app.js +++ b/packages/gatsby/cache-dir/app.js @@ -5,9 +5,8 @@ import domReady from "@mikaelkristiansson/domready" import socketIo from "./socketIo" import emitter from "./emitter" import { apiRunner, apiRunnerAsync } from "./api-runner-browser" -import loader, { setApiRunnerForLoader, postInitialRenderWork } from "./loader" +import loader, { setApiRunnerForLoader } from "./loader" import syncRequires from "./sync-requires" -import pages from "./pages.json" window.___emitter = emitter setApiRunnerForLoader(apiRunner) @@ -49,18 +48,16 @@ apiRunnerAsync(`onClientEntry`).then(() => { ReactDOM.render )[0] - loader.addPagesArray(pages) loader.addDevRequires(syncRequires) Promise.all([ - loader.getResourcesForPathname(`/dev-404-page/`), - loader.getResourcesForPathname(`/404.html`), - loader.getResourcesForPathname(window.location.pathname), + loader.loadPage(`/dev-404-page/`), + loader.loadPage(`/404.html`), + loader.loadPage(window.location.pathname), ]).then(() => { const preferDefault = m => (m && m.default) || m let Root = preferDefault(require(`./root`)) domReady(() => { renderer(, rootElement, () => { - postInitialRenderWork() apiRunner(`onInitialClientRender`) }) }) diff --git a/packages/gatsby/cache-dir/ensure-resources.js b/packages/gatsby/cache-dir/ensure-resources.js index 0c37dd50333d6..080d0ec11d536 100644 --- a/packages/gatsby/cache-dir/ensure-resources.js +++ b/packages/gatsby/cache-dir/ensure-resources.js @@ -17,7 +17,7 @@ class EnsureResources extends React.Component { this.state = { location: { ...location }, - pageResources: loader.getResourcesForPathnameSync(location.pathname), + pageResources: loader.getPageOr404(location.pathname), } } @@ -34,9 +34,7 @@ class EnsureResources extends React.Component { static getDerivedStateFromProps({ location }, prevState) { if (prevState.location !== location) { - const pageResources = loader.getResourcesForPathnameSync( - location.pathname - ) + const pageResources = loader.getPageOr404(location.pathname) return { pageResources, @@ -62,7 +60,7 @@ class EnsureResources extends React.Component { retryResources(nextProps) { const { pathname } = nextProps.location - if (!loader.getResourcesForPathnameSync(pathname)) { + if (!loader.getPage(pathname)) { // Store the previous and next location before resolving resources const prevLocation = this.props.location this.nextLocation = nextProps.location @@ -70,7 +68,7 @@ class EnsureResources extends React.Component { // Page resources won't be set in cases where the browser back button // or forward button is pushed as we can't wait as normal for resources // to load before changing the page. - loader.getResourcesForPathname(pathname).then(pageResources => { + loader.loadPageOr404(pathname).then(pageResources => { // The page may have changed since we started this, in which case doesn't update if (this.nextLocation !== nextProps.location) { return diff --git a/packages/gatsby/cache-dir/find-page.js b/packages/gatsby/cache-dir/find-page.js deleted file mode 100644 index f25078f4f6b7b..0000000000000 --- a/packages/gatsby/cache-dir/find-page.js +++ /dev/null @@ -1,55 +0,0 @@ -// TODO add tests especially for handling prefixed links. -import { match as matchPath } from "@reach/router/lib/utils" -import stripPrefix from "./strip-prefix" - -const pageCache = {} - -export default (pages, pathPrefix = ``) => rawPathname => { - let pathname = decodeURIComponent(rawPathname) - - // Remove the pathPrefix from the pathname. - let trimmedPathname = stripPrefix(pathname, pathPrefix) - - // Remove any hashfragment - if (trimmedPathname.split(`#`).length > 1) { - trimmedPathname = trimmedPathname - .split(`#`) - .slice(0, -1) - .join(``) - } - - // Remove search query - if (trimmedPathname.split(`?`).length > 1) { - trimmedPathname = trimmedPathname - .split(`?`) - .slice(0, -1) - .join(``) - } - - if (pageCache[trimmedPathname]) { - return pageCache[trimmedPathname] - } - - let foundPage - // Array.prototype.find is not supported in IE so we use this somewhat odd - // work around. - pages.some(page => { - let pathToMatch = page.matchPath ? page.matchPath : page.path - if (matchPath(pathToMatch, trimmedPathname)) { - foundPage = page - pageCache[trimmedPathname] = page - return true - } - - // Finally, try and match request with default document. - if (matchPath(`${page.path}index.html`, trimmedPathname)) { - foundPage = page - pageCache[trimmedPathname] = page - return true - } - - return false - }) - - return foundPage -} diff --git a/packages/gatsby/cache-dir/json-store.js b/packages/gatsby/cache-dir/json-store.js index 8bec0d5b40c51..147198b193313 100644 --- a/packages/gatsby/cache-dir/json-store.js +++ b/packages/gatsby/cache-dir/json-store.js @@ -1,6 +1,7 @@ import React from "react" import PageRenderer from "./page-renderer" +import normalizePagePath from "./normalize-page-path" import { StaticQueryContext } from "gatsby" import { getStaticQueryData, @@ -22,7 +23,7 @@ if (process.env.NODE_ENV === `production`) { const getPathFromProps = props => props.pageResources && props.pageResources.page - ? props.pageResources.page.path + ? normalizePagePath(props.pageResources.page.path) : undefined class JSONStore extends React.Component { @@ -74,8 +75,8 @@ class JSONStore extends React.Component { return ( this.props.location !== nextProps.location || this.state.path !== nextState.path || - this.state.pageQueryData[nextState.path] !== - nextState.pageQueryData[nextState.path] || + this.state.pageQueryData[normalizePagePath(nextState.path)] !== + nextState.pageQueryData[normalizePagePath(nextState.path)] || this.state.staticQueryData !== nextState.staticQueryData ) } diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index 5a9dec232489e..3423ec76f4b82 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -1,166 +1,157 @@ -import pageFinderFactory from "./find-page" import emitter from "./emitter" import prefetchHelper from "./prefetch" +import { match } from "@reach/router/lib/utils" +import normalizePagePath from "./normalize-page-path" +import stripPrefix from "./strip-prefix" +// Generated during bootstrap +import matchPaths from "./match-paths.json" const preferDefault = m => (m && m.default) || m -let devGetPageData -let inInitialRender = true -let hasFetched = Object.create(null) +const pageNotFoundPaths = new Set() + +let apiRunner let syncRequires = {} let asyncRequires = {} -let jsonDataPaths = {} -let fetchHistory = [] -let fetchingPageResourceMapPromise = null -let fetchedPageResourceMap = false -/** - * Indicate if pages manifest is loaded - * - in production it is split to separate "pages-manifest" chunk that need to be lazy loaded, - * - in development it is part of single "common" chunk and is available from the start. - */ -let hasPageResourceMap = process.env.NODE_ENV !== `production` -let apiRunner -const failedPaths = {} -const MAX_HISTORY = 5 -const jsonPromiseStore = {} +const fetchedPageData = {} +const pageDatas = {} +const fetchPromiseStore = {} +let devGetPageData if (process.env.NODE_ENV !== `production`) { devGetPageData = require(`./socketIo`).getPageData } -/** - * Fetch resource map (pages data and paths to json files with results of - * queries) - */ -const fetchPageResourceMap = () => { - if (!fetchingPageResourceMapPromise) { - fetchingPageResourceMapPromise = new Promise(resolve => { - asyncRequires - .data() - .then(({ pages, dataPaths }) => { - // TODO — expose proper way to access this data from plugins. - // Need to come up with an API for plugins to access - // site info. - window.___dataPaths = dataPaths - queue.addPagesArray(pages) - queue.addDataPaths(dataPaths) - hasPageResourceMap = true - resolve((fetchedPageResourceMap = true)) - }) - .catch(e => { - console.warn( - `Failed to fetch pages manifest. Gatsby will reload on next navigation.` - ) - // failed to grab pages metadata - // for now let's just resolve this - on navigation this will cause missing resources - // and will trigger page reload and then it will retry - // this can happen with service worker updates when webpack manifest points to old - // chunk that no longer exists on server - resolve((fetchedPageResourceMap = true)) - }) - }) +// Cache for `cleanAndFindPath()`. In case `match-paths.json` is large +const cleanAndFindPathCache = {} + +const findMatchPath = (matchPaths, trimmedPathname) => { + for (const { matchPath, path } of matchPaths) { + if (match(matchPath, trimmedPathname)) { + return path + } } - return fetchingPageResourceMapPromise + return null } -const createJsonURL = jsonName => `${__PATH_PREFIX__}/static/d/${jsonName}.json` -const createComponentUrls = componentChunkName => - window.___chunkMapping[componentChunkName].map( - chunk => __PATH_PREFIX__ + chunk - ) +// Given a raw URL path, returns the cleaned version of it (trim off +// `#` and query params), or if it matches an entry in +// `match-paths.json`, its matched path is returned +// +// E.g `/foo?bar=far` => `/foo` +// +// Or if `match-paths.json` contains `{ "/foo*": "/page1", ...}`, then +// `/foo?bar=far` => `/page1` +const cleanAndFindPath = rawPathname => { + let pathname = decodeURIComponent(rawPathname) + // Remove the pathPrefix from the pathname. + let trimmedPathname = stripPrefix(pathname, __PATH_PREFIX__) + // Remove any hashfragment + if (trimmedPathname.split(`#`).length > 1) { + trimmedPathname = trimmedPathname + .split(`#`) + .slice(0, -1) + .join(``) + } -const fetchResource = resourceName => { - // Find resource - let resourceFunction - if (resourceName.slice(0, 12) === `component---`) { - resourceFunction = asyncRequires.components[resourceName] - } else { - if (resourceName in jsonPromiseStore) { - resourceFunction = () => jsonPromiseStore[resourceName] + // Remove search query + if (trimmedPathname.split(`?`).length > 1) { + trimmedPathname = trimmedPathname + .split(`?`) + .slice(0, -1) + .join(``) + } + if (cleanAndFindPathCache[trimmedPathname]) { + return cleanAndFindPathCache[trimmedPathname] + } + + let foundPath = findMatchPath(matchPaths, trimmedPathname) + if (!foundPath) { + if (trimmedPathname === `/index.html`) { + foundPath = `/` } else { - resourceFunction = () => { - const fetchPromise = new Promise((resolve, reject) => { - const url = createJsonURL(jsonDataPaths[resourceName]) - const req = new XMLHttpRequest() - req.open(`GET`, url, true) - req.withCredentials = true - req.onreadystatechange = () => { - if (req.readyState == 4) { - if (req.status === 200) { - resolve(JSON.parse(req.responseText)) - } else { - delete jsonPromiseStore[resourceName] - reject() - } - } - } - req.send(null) - }) - jsonPromiseStore[resourceName] = fetchPromise - return fetchPromise - } + foundPath = trimmedPathname } } + foundPath = normalizePagePath(foundPath) + cleanAndFindPathCache[trimmedPathname] = foundPath + return foundPath +} - // Download the resource - hasFetched[resourceName] = true - return new Promise(resolve => { - const fetchPromise = resourceFunction() - let failed = false - return fetchPromise - .catch(() => { - failed = true - }) - .then(component => { - fetchHistory.push({ - resource: resourceName, - succeeded: !failed, - }) - - fetchHistory = fetchHistory.slice(-MAX_HISTORY) - - resolve(component) - }) +const cachedFetch = (resourceName, fetchFn) => { + if (resourceName in fetchPromiseStore) { + return fetchPromiseStore[resourceName] + } + const promise = fetchFn(resourceName) + fetchPromiseStore[resourceName] = promise + return promise.catch(err => { + delete fetchPromiseStore[resourceName] + return err }) } -const prefetchResource = resourceName => { - if (resourceName.slice(0, 12) === `component---`) { - return Promise.all( - createComponentUrls(resourceName).map(url => prefetchHelper(url)) - ) +const doFetch = url => + new Promise((resolve, reject) => { + const req = new XMLHttpRequest() + req.open(`GET`, url, true) + req.withCredentials = true + req.onreadystatechange = () => { + if (req.readyState == 4) { + resolve(req) + } + } + req.send(null) + }) + +const handlePageDataResponse = (path, req) => { + fetchedPageData[path] = true + if (req.status === 200) { + const contentType = req.getResponseHeader(`content-type`) + // Since we don't know if a URL is a page or not until we make a + // request to the server, the response could be anything. E.g an + // index.html. So we have to double check that response is + // actually a proper JSON file. If it isn't, then it's not a page + // and we can infer that the requested page doesn't exist + if (!contentType || !contentType.startsWith(`application/json`)) { + pageNotFoundPaths.add(path) + return null + } else { + const pageData = JSON.parse(req.responseText) + pageDatas[path] = pageData + return pageData + } + } else if (req.status === 404) { + pageNotFoundPaths.add(path) + return null } else { - const url = createJsonURL(jsonDataPaths[resourceName]) - return prefetchHelper(url) + throw new Error(`error fetching page`) } } -const getResourceModule = resourceName => - fetchResource(resourceName).then(preferDefault) +const fetchPageData = path => { + const url = createPageDataUrl(path) + return cachedFetch(url, doFetch).then(req => + handlePageDataResponse(path, req) + ) +} -const appearsOnLine = () => { - const isOnLine = navigator.onLine - if (typeof isOnLine === `boolean`) { - return isOnLine - } +const createComponentUrls = componentChunkName => + window.___chunkMapping[componentChunkName].map( + chunk => __PATH_PREFIX__ + chunk + ) - // If no navigator.onLine support assume onLine if any of last N fetches succeeded - const succeededFetch = fetchHistory.find(entry => entry.succeeded) - return !!succeededFetch -} +const fetchComponent = chunkName => asyncRequires.components[chunkName]() -const handleResourceLoadError = (path, message) => { - if (!failedPaths[path]) { - failedPaths[path] = message - } +const stripSurroundingSlashes = s => { + s = s[0] === `/` ? s.slice(1) : s + s = s.endsWith(`/`) ? s.slice(0, -1) : s + return s +} - if ( - appearsOnLine() && - window.location.pathname.replace(/\/$/g, ``) !== path.replace(/\/$/g, ``) - ) { - window.location.pathname = path - } +const createPageDataUrl = path => { + const fixedPath = path === `/` ? `index` : stripSurroundingSlashes(path) + return `${__PATH_PREFIX__}/page-data/${fixedPath}/page-data.json` } const onPrefetchPathname = pathname => { @@ -170,57 +161,44 @@ const onPrefetchPathname = pathname => { } } -const onPostPrefetchPathname = pathname => { - if (!prefetchCompleted[pathname]) { - apiRunner(`onPostPrefetchPathname`, { pathname }) - prefetchCompleted[pathname] = true - } -} - -/** - * Check if we should fallback to resources for 404 page if resources for a page are not found - * - * We can't do that when we don't have full pages manifest - we don't know if page exist or not if we don't have it. - * We also can't do that on initial render / mount in case we just can't load resources needed for first page. - * Not falling back to 404 resources will cause "EnsureResources" component to handle scenarios like this with - * potential reload - * @param {string} path Path to a page - */ -const shouldFallbackTo404Resources = path => - (hasPageResourceMap || inInitialRender) && path !== `/404.html` - // Note we're not actively using the path data atm. There // could be future optimizations however around trying to ensure // we load all resources for likely-to-be-visited paths. // let pathArray = [] // let pathCount = {} -let findPage let pathScriptsCache = {} let prefetchTriggered = {} let prefetchCompleted = {} let disableCorePrefetching = false +const onPostPrefetch = ({ path, resourceUrls }) => { + if (!prefetchCompleted[path]) { + apiRunner(`onPostPrefetch`, { path, resourceUrls }) + prefetchCompleted[path] = true + } +} + +const loadComponent = componentChunkName => { + if (process.env.NODE_ENV !== `production`) { + return Promise.resolve(syncRequires.components[componentChunkName]) + } else { + return cachedFetch(componentChunkName, fetchComponent).then(preferDefault) + } +} + const queue = { - addPagesArray: newPages => { - findPage = pageFinderFactory(newPages, __BASE_PATH__) - }, addDevRequires: devRequires => { syncRequires = devRequires }, addProdRequires: prodRequires => { asyncRequires = prodRequires }, - addDataPaths: dataPaths => { - jsonDataPaths = dataPaths - }, // Hovering on a link is a very strong indication the user is going to // click on it soon so let's start prefetching resources for this // pathname. - hovering: path => { - queue.getResourcesForPathname(path) - }, - enqueue: path => { + hovering: path => queue.loadPage(path), + enqueue: rawPath => { if (!apiRunner) console.error(`Run setApiRunnerForLoader() before enqueing paths`) @@ -236,7 +214,7 @@ const queue = { // Tell plugins with custom prefetching logic that they should start // prefetching this path. - onPrefetchPathname(path) + onPrefetchPathname(rawPath) // If a plugin has disabled core prefetching, stop now. if (disableCorePrefetching.some(a => a)) { @@ -244,193 +222,148 @@ const queue = { } // Check if the page exists. - let page = findPage(path) - - // In production, we lazy load page metadata. If that - // hasn't been fetched yet, start fetching it now. - if ( - process.env.NODE_ENV === `production` && - !page && - !fetchedPageResourceMap - ) { - // If page wasn't found check and we didn't fetch resources map for - // all pages, wait for fetch to complete and try find page again - return fetchPageResourceMap().then(() => queue.enqueue(path)) - } + let realPath = cleanAndFindPath(rawPath) - if (!page) { - return false + if (pageDatas[realPath]) { + return true } if ( process.env.NODE_ENV !== `production` && process.env.NODE_ENV !== `test` ) { - devGetPageData(page.path) + // Ensure latest version of page data is in the JSON store + devGetPageData(realPath) } - // Prefetch resources. if (process.env.NODE_ENV === `production`) { - Promise.all([ - prefetchResource(page.jsonName), - prefetchResource(page.componentChunkName), - ]).then(() => { - // Tell plugins the path has been successfully prefetched - onPostPrefetchPathname(path) - }) + const pageDataUrl = createPageDataUrl(realPath) + prefetchHelper(pageDataUrl) + .then(() => + // This was just prefetched, so will return a response from + // the cache instead of making another request to the server + fetchPageData(realPath) + ) + .then(pageData => { + // Tell plugins the path has been successfully prefetched + const chunkName = pageData.componentChunkName + const componentUrls = createComponentUrls(chunkName) + return Promise.all(componentUrls.map(prefetchHelper)).then(() => { + const resourceUrls = [pageDataUrl].concat(componentUrls) + onPostPrefetch({ + path: rawPath, + resourceUrls, + }) + }) + }) } return true }, - getPage: pathname => findPage(pathname), + isPageNotFound: pathname => pageNotFoundPaths.has(cleanAndFindPath(pathname)), - getResourceURLsForPathname: path => { - const page = findPage(path) - if (page) { - return [ - ...createComponentUrls(page.componentChunkName), - createJsonURL(jsonDataPaths[page.jsonName]), - ] - } else { - return null + loadPageData: rawPath => { + const realPath = cleanAndFindPath(rawPath) + if (queue.isPageNotFound(realPath)) { + return Promise.resolve(null) } - }, - - getResourcesForPathnameSync: path => { - const page = findPage(path) - if (page) { - return pathScriptsCache[page.path] - } else if (shouldFallbackTo404Resources(path)) { - return queue.getResourcesForPathnameSync(`/404.html`) - } else { - return null - } - }, - - // Get resources (code/data) for a path. Fetches metdata first - // if necessary and then the code/data bundles. Used for prefetching - // and getting resources for page changes. - getResourcesForPathname: path => - new Promise((resolve, reject) => { - // Production code path - if (failedPaths[path]) { - handleResourceLoadError( - path, - `Previously detected load failure for "${path}"` - ) - reject() - return - } - const page = findPage(path) - - // In production, we lazy load page metadata. If that - // hasn't been fetched yet, start fetching it now. - if ( - !page && - !fetchedPageResourceMap && - process.env.NODE_ENV === `production` - ) { - // If page wasn't found check and we didn't fetch resources map for - // all pages, wait for fetch to complete and try to get resources again - fetchPageResourceMap().then(() => - resolve(queue.getResourcesForPathname(path)) - ) - return - } - - if (!page) { - if (shouldFallbackTo404Resources(path)) { - console.log(`A page wasn't found for "${path}"`) - - // Preload the custom 404 page - resolve(queue.getResourcesForPathname(`/404.html`)) - return + if (!fetchedPageData[realPath]) { + return fetchPageData(realPath).then(pageData => { + if (process.env.NODE_ENV !== `production`) { + devGetPageData(realPath) } - - resolve() - return - } - - // Use the path from the page so the pathScriptsCache uses - // the normalized path. - path = page.path - - // Check if it's in the cache already. - if (pathScriptsCache[path]) { - emitter.emit(`onPostLoadPageResources`, { - page, - pageResources: pathScriptsCache[path], - }) - resolve(pathScriptsCache[path]) - return - } - - // Nope, we need to load resource(s) - emitter.emit(`onPreLoadPageResources`, { - path, + return queue.loadPageData(rawPath) }) + } + return Promise.resolve(pageDatas[realPath]) + }, - // In development we know the code is loaded already - // so we just return with it immediately. - if (process.env.NODE_ENV !== `production`) { - const pageResources = { - component: syncRequires.components[page.componentChunkName], - page, + loadPage: rawPath => + queue + .loadPageData(rawPath) + .then(pageData => { + // If no page was found, then preload the 404.html + if (pageData === null && rawPath !== `/404.html`) { + return queue.loadPage(`/404.html`).then(() => null) } - - // Add to the cache. - pathScriptsCache[path] = pageResources - devGetPageData(page.path).then(pageData => { - emitter.emit(`onPostLoadPageResources`, { - page, - pageResources, - }) - // Tell plugins the path has been successfully prefetched - onPostPrefetchPathname(path) - - resolve(pageResources) - }) - } else { - Promise.all([ - getResourceModule(page.componentChunkName), - getResourceModule(page.jsonName), - ]).then(([component, json]) => { - if (!(component && json)) { - resolve(null) - return + // Otherwise go ahead and load the page's component + return loadComponent(pageData.componentChunkName).then(component => { + const page = { + componentChunkName: pageData.componentChunkName, + path: pageData.path, } + const jsonData = pageData.result + const pageResources = { component, - json, + json: jsonData, page, } - pageResources.page.jsonURL = createJsonURL( - jsonDataPaths[page.jsonName] - ) - pathScriptsCache[path] = pageResources - resolve(pageResources) + pathScriptsCache[cleanAndFindPath(rawPath)] = pageResources emitter.emit(`onPostLoadPageResources`, { - page, + page: pageResources, pageResources, }) + if (process.env.NODE_ENV === `production`) { + const pageDataUrl = createPageDataUrl(cleanAndFindPath(rawPath)) + const componentUrls = createComponentUrls( + pageData.componentChunkName + ) + const resourceUrls = [pageDataUrl].concat(componentUrls) + onPostPrefetch({ + path: rawPath, + resourceUrls, + }) + } - // Tell plugins the path has been successfully prefetched - onPostPrefetchPathname(path) + return pageResources }) - } - }), + }) + .catch(() => null), + + loadPageOr404: rawPath => + queue + .loadPage(rawPath) + .then(result => + result === null && rawPath !== `/404.html` + ? queue.getPage(`/404.html`) + : null + ), + + getPage: rawPath => pathScriptsCache[cleanAndFindPath(rawPath)], + + getPageOr404: rawPath => { + const page = queue.getPage(rawPath) + if (page) { + return page + } else if (rawPath !== `/404.html`) { + return queue.getPage(`/404.html`) + } else { + return null + } + }, + + // Deprecated April 2019. Query results used to be in a separate + // file, but are now included in the page-data.json, which is + // already loaded into the browser by the time this function is + // called. Use the resource URLs passed in `onPostPrefetch` instead. + getResourceURLsForPathname: path => { + const pageData = queue.getPage(path) + if (pageData) { + // Original implementation also concatenated the jsonDataPath + // for the page + return createComponentUrls(pageData.componentChunkName) + } else { + return null + } + }, } export const postInitialRenderWork = () => { - inInitialRender = false - if (process.env.NODE_ENV === `production`) { - // We got all resources needed for first mount, - // we can fetch resources for all pages. - fetchPageResourceMap() - } + console.warn(`Warning: postInitialRenderWork is deprecated. It is now a noop`) } export const setApiRunnerForLoader = runner => { @@ -439,9 +372,32 @@ export const setApiRunnerForLoader = runner => { } export const publicLoader = { - getResourcesForPathname: queue.getResourcesForPathname, - getResourceURLsForPathname: queue.getResourceURLsForPathname, - getResourcesForPathnameSync: queue.getResourcesForPathnameSync, + // Deprecated methods. As far as we're aware, these are only used by + // core gatsby and the offline plugin, however there's a very small + // chance they're called by others. + getResourcesForPathname: rawPath => { + console.warn( + `Warning: getResourcesForPathname is deprecated. Use loadPage instead` + ) + return queue.loadPage(rawPath) + }, + getResourcesForPathnameSync: rawPath => { + console.warn( + `Warning: getResourcesForPathnameSync is deprecated. Use getPage instead` + ) + return queue.getPage(rawPath) + }, + getResourceURLsForPathname: pathname => { + console.warn( + `Warning: getResourceURLsForPathname is deprecated. Use onPostPrefetch instead` + ) + return queue.getResourceURLsForPathname + }, + + // Real methods + loadPage: queue.loadPage, + getPage: queue.getPage, + getPageOr404: queue.getPageOr404, } export default queue diff --git a/packages/gatsby/cache-dir/navigation.js b/packages/gatsby/cache-dir/navigation.js index 6289cc2d3f8a1..85808d6d10251 100644 --- a/packages/gatsby/cache-dir/navigation.js +++ b/packages/gatsby/cache-dir/navigation.js @@ -18,7 +18,7 @@ function maybeRedirect(pathname) { if (redirect != null) { if (process.env.NODE_ENV !== `production`) { - const pageResources = loader.getResourcesForPathnameSync(pathname) + const pageResources = loader.getPage(pathname) if (pageResources != null) { console.error( @@ -81,7 +81,7 @@ const navigate = (to, options = {}) => { }) }, 1000) - loader.getResourcesForPathname(pathname).then(pageResources => { + loader.loadPage(pathname).then(pageResources => { reachNavigate(to, options) clearTimeout(timeoutId) }) diff --git a/packages/gatsby/cache-dir/normalize-page-path.js b/packages/gatsby/cache-dir/normalize-page-path.js new file mode 100644 index 0000000000000..e3aa70b2f45c5 --- /dev/null +++ b/packages/gatsby/cache-dir/normalize-page-path.js @@ -0,0 +1,12 @@ +export default path => { + if (path === undefined) { + return path + } + if (path === `/`) { + return `/` + } + if (path.charAt(path.length - 1) === `/`) { + return path.slice(0, -1) + } + return path +} diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js index 90d8f871289cc..e1a77480ddc73 100644 --- a/packages/gatsby/cache-dir/production-app.js +++ b/packages/gatsby/cache-dir/production-app.js @@ -2,7 +2,6 @@ import { apiRunner, apiRunnerAsync } from "./api-runner-browser" import React, { createElement } from "react" import ReactDOM from "react-dom" import { Router, navigate } from "@reach/router" -import { match } from "@reach/router/lib/utils" import { ScrollContext } from "gatsby-react-router-scroll" import domReady from "@mikaelkristiansson/domready" import { @@ -13,15 +12,13 @@ import { import emitter from "./emitter" import PageRenderer from "./page-renderer" import asyncRequires from "./async-requires" -import loader, { setApiRunnerForLoader, postInitialRenderWork } from "./loader" +import loader, { setApiRunnerForLoader } from "./loader" import EnsureResources from "./ensure-resources" window.asyncRequires = asyncRequires window.___emitter = emitter window.___loader = loader -loader.addPagesArray([window.page]) -loader.addDataPaths({ [window.page.jsonName]: window.dataPath }) loader.addProdRequires(asyncRequires) setApiRunnerForLoader(apiRunner) @@ -61,28 +58,25 @@ apiRunnerAsync(`onClientEntry`).then(() => { } } - const { page, location: browserLoc } = window + const { pagePath, location: browserLoc } = window if ( // Make sure the window.page object is defined - page && + pagePath && // The canonical path doesn't match the actual path (i.e. the address bar) - __BASE_PATH__ + page.path !== browserLoc.pathname && - // ...and if matchPage is specified, it also doesn't match the actual path - (!page.matchPath || - !match(__BASE_PATH__ + page.matchPath, browserLoc.pathname)) && + __BASE_PATH__ + pagePath !== browserLoc.pathname && // Ignore 404 pages, since we want to keep the same URL - page.path !== `/404.html` && - !page.path.match(/^\/404\/?$/) && + pagePath !== `/404.html` && + !pagePath.match(/^\/404\/?$/) && // Also ignore the offline shell (since when using the offline plugin, all // pages have this canonical path) - !page.path.match(/^\/offline-plugin-app-shell-fallback\/?$/) + !pagePath.match(/^\/offline-plugin-app-shell-fallback\/?$/) ) { - navigate(__BASE_PATH__ + page.path + browserLoc.search + browserLoc.hash, { + navigate(__BASE_PATH__ + pagePath + browserLoc.search + browserLoc.hash, { replace: true, }) } - loader.getResourcesForPathname(browserLoc.pathname).then(() => { + loader.loadPage(browserLoc.pathname).then(() => { const Root = () => createElement( Router, @@ -116,7 +110,6 @@ apiRunnerAsync(`onClientEntry`).then(() => { ? document.getElementById(`___gatsby`) : void 0, () => { - postInitialRenderWork() apiRunner(`onInitialClientRender`) } ) diff --git a/packages/gatsby/cache-dir/public-page-renderer-dev.js b/packages/gatsby/cache-dir/public-page-renderer-dev.js index 6bcfd16d4bd20..56abdba349228 100644 --- a/packages/gatsby/cache-dir/public-page-renderer-dev.js +++ b/packages/gatsby/cache-dir/public-page-renderer-dev.js @@ -5,7 +5,7 @@ import loader from "./loader" import JSONStore from "./json-store" const DevPageRenderer = ({ location }) => { - const pageResources = loader.getResourcesForPathnameSync(location.pathname) + const pageResources = loader.getPage(location.pathname) return React.createElement(JSONStore, { location, pageResources, diff --git a/packages/gatsby/cache-dir/public-page-renderer-prod.js b/packages/gatsby/cache-dir/public-page-renderer-prod.js index 195567cd6241b..55bafa0886b1b 100644 --- a/packages/gatsby/cache-dir/public-page-renderer-prod.js +++ b/packages/gatsby/cache-dir/public-page-renderer-prod.js @@ -5,11 +5,10 @@ import InternalPageRenderer from "./page-renderer" import loader from "./loader" const ProdPageRenderer = ({ location }) => { - const pageResources = loader.getResourcesForPathnameSync(location.pathname) + const pageResources = loader.getPageOr404(location.pathname) if (!pageResources) { return null } - return React.createElement(InternalPageRenderer, { location, pageResources, diff --git a/packages/gatsby/cache-dir/root.js b/packages/gatsby/cache-dir/root.js index c19a134f30bcb..7aceccb591437 100644 --- a/packages/gatsby/cache-dir/root.js +++ b/packages/gatsby/cache-dir/root.js @@ -37,11 +37,7 @@ class RouteHandler extends React.Component { render() { let { location } = this.props - // check if page exists - in dev pages are sync loaded, it's safe to use - // loader.getPage - let page = loader.getPage(location.pathname) - - if (page) { + if (!loader.isPageNotFound(location.pathname)) { return ( {locationAndPageResources => ( @@ -56,30 +52,26 @@ class RouteHandler extends React.Component { )} ) - } else { - const dev404PageResources = loader.getResourcesForPathnameSync( - `/dev-404-page/` - ) - const real404PageResources = loader.getResourcesForPathnameSync( - `/404.html` - ) - let custom404 - if (real404PageResources) { - custom404 = ( - - ) - } + } - return ( - - - + const dev404PageResources = loader.getPage(`/dev-404-page`) + const real404PageResources = loader.getPage(`/404.html`) + let custom404 + if (real404PageResources) { + custom404 = ( + ) } + + return ( + + + + ) } } diff --git a/packages/gatsby/cache-dir/socketIo.js b/packages/gatsby/cache-dir/socketIo.js index 978f7db8966d1..c645499a31e42 100644 --- a/packages/gatsby/cache-dir/socketIo.js +++ b/packages/gatsby/cache-dir/socketIo.js @@ -1,4 +1,5 @@ import { reportError, clearError } from "./error-overlay-handler" +import normalizePagePath from "./normalize-page-path" let socket = null @@ -18,10 +19,16 @@ export default function socketIo() { // eslint-disable-next-line no-undef socket = io() - const didDataChange = (msg, queryData) => - !(msg.payload.id in queryData) || - JSON.stringify(msg.payload.result) !== - JSON.stringify(queryData[msg.payload.id]) + const didDataChange = (msg, queryData) => { + const id = + msg.type === `staticQueryResult` + ? msg.payload.id + : normalizePagePath(msg.payload.id) + return ( + !(id in queryData) || + JSON.stringify(msg.payload.result) !== JSON.stringify(queryData[id]) + ) + } socket.on(`message`, msg => { if (msg.type === `staticQueryResult`) { @@ -35,7 +42,7 @@ export default function socketIo() { if (didDataChange(msg, pageQueryData)) { pageQueryData = { ...pageQueryData, - [msg.payload.id]: msg.payload.result, + [normalizePagePath(msg.payload.id)]: msg.payload.result, } } } else if (msg.type === `overlayError`) { @@ -61,6 +68,7 @@ export default function socketIo() { const inFlightGetPageDataPromiseCache = {} function getPageData(pathname) { + pathname = normalizePagePath(pathname) if (inFlightGetPageDataPromiseCache[pathname]) { return inFlightGetPageDataPromiseCache[pathname] } else { @@ -70,7 +78,10 @@ function getPageData(pathname) { resolve(pageQueryData[pathname]) } else { const onPageDataCallback = msg => { - if (msg.type === `pageQueryResult` && msg.payload.id === pathname) { + if ( + msg.type === `pageQueryResult` && + normalizePagePath(msg.payload.id) === pathname + ) { socket.off(`message`, onPageDataCallback) delete inFlightGetPageDataPromiseCache[pathname] resolve(pageQueryData[pathname]) diff --git a/packages/gatsby/cache-dir/static-entry.js b/packages/gatsby/cache-dir/static-entry.js index dfb4e8f94a352..5e09ad0ab56bd 100644 --- a/packages/gatsby/cache-dir/static-entry.js +++ b/packages/gatsby/cache-dir/static-entry.js @@ -7,13 +7,8 @@ const { get, merge, isObject, flatten, uniqBy } = require(`lodash`) const apiRunner = require(`./api-runner-ssr`) const syncRequires = require(`./sync-requires`) -const { dataPaths, pages } = require(`./data.json`) const { version: gatsbyVersion } = require(`gatsby/package.json`) -// Speed up looking up pages. -const pagesObjectMap = new Map() -pages.forEach(p => pagesObjectMap.set(p.path, p)) - const stats = JSON.parse( fs.readFileSync(`${process.cwd()}/public/webpack.stats.json`, `utf-8`) ) @@ -45,7 +40,20 @@ try { Html = Html && Html.__esModule ? Html.default : Html -const getPage = path => pagesObjectMap.get(path) +const getPageDataPath = path => { + const fixedPagePath = path === `/` ? `index` : path + return join(`page-data`, fixedPagePath, `page-data.json`) +} + +const getPageDataUrl = pagePath => { + const pageDataPath = getPageDataPath(pagePath) + return `${__PATH_PREFIX__}/${pageDataPath}` +} + +const getPageDataFile = pagePath => { + const pageDataPath = getPageDataPath(pagePath) + return join(process.cwd(), `public`, pageDataPath) +} const createElement = React.createElement @@ -122,33 +130,20 @@ export default (pagePath, callback) => { postBodyComponents = sanitizeComponents(components) } - const page = getPage(pagePath) - - let dataAndContext = {} - if (page.jsonName in dataPaths) { - const pathToJsonData = join( - process.cwd(), - `/public/static/d`, - `${dataPaths[page.jsonName]}.json` - ) - try { - dataAndContext = JSON.parse(fs.readFileSync(pathToJsonData)) - } catch (e) { - console.log(`error`, pathToJsonData, e) - process.exit() - } - } + const pageDataRaw = fs.readFileSync(getPageDataFile(pagePath)) + const pageData = JSON.parse(pageDataRaw) + const pageDataUrl = getPageDataUrl(pagePath) + const { componentChunkName } = pageData class RouteHandler extends React.Component { render() { const props = { ...this.props, - ...dataAndContext, - pathContext: dataAndContext.pageContext, + ...pageData.result, } const pageElement = createElement( - syncRequires.components[page.componentChunkName], + syncRequires.components[componentChunkName], props ) @@ -212,7 +207,7 @@ export default (pagePath, callback) => { // Create paths to scripts let scriptsAndStyles = flatten( - [`app`, page.componentChunkName].map(s => { + [`app`, componentChunkName].map(s => { const fetchKey = `assetsByChunkName[${s}]` let chunks = get(stats, fetchKey) @@ -287,16 +282,13 @@ export default (pagePath, callback) => { ) }) - if (page.jsonName in dataPaths) { - const dataPath = `${__PATH_PREFIX__}/static/d/${ - dataPaths[page.jsonName] - }.json` + if (pageData) { headComponents.push( ) @@ -334,18 +326,14 @@ export default (pagePath, callback) => { }) // Add page metadata for the current page - const windowData = `/**/` + const windowPagePath = `/**/` postBodyComponents.push(
div3
div2
div1
"`; -exports[`static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"
"`; +exports[`static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"
"`; -exports[`static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"
div3
div2
div1
"`; +exports[`static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"
div3
div2
div1
"`; -exports[`static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"
div3
div2
div1
"`; +exports[`static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"
div3
div2
div1
"`; diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js index 0f5253ccf1c08..1796f9913d667 100644 --- a/packages/gatsby/cache-dir/__tests__/static-entry.js +++ b/packages/gatsby/cache-dir/__tests__/static-entry.js @@ -37,6 +37,7 @@ const MOCK_FILE_INFO = { componentChunkName: `page-component---src-pages-test-js`, path: `/about/`, jsonName: `about.json`, + webpackCompilationHash: `1234567890abcdef1234`, }), } diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index 3423ec76f4b82..6584cc30bf8b8 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -292,6 +292,7 @@ const queue = { const page = { componentChunkName: pageData.componentChunkName, path: pageData.path, + webpackCompilationHash: pageData.webpackCompilationHash, } const jsonData = pageData.result diff --git a/packages/gatsby/cache-dir/navigation.js b/packages/gatsby/cache-dir/navigation.js index 85808d6d10251..9201a89855c72 100644 --- a/packages/gatsby/cache-dir/navigation.js +++ b/packages/gatsby/cache-dir/navigation.js @@ -82,6 +82,28 @@ const navigate = (to, options = {}) => { }, 1000) loader.loadPage(pathname).then(pageResources => { + // If the loaded page has a different compilation hash to the + // window, then a rebuild has occurred on the server. Reload. + if (process.env.NODE_ENV === `production` && pageResources) { + if ( + pageResources.page.webpackCompilationHash !== + window.___webpackCompilationHash + ) { + // Purge plugin-offline cache + if ( + `serviceWorker` in navigator && + navigator.serviceWorker.controller !== null && + navigator.serviceWorker.controller.state === `activated` + ) { + navigator.serviceWorker.controller.postMessage({ + gatsbyApi: `resetWhitelist`, + }) + } + + console.log(`Site has changed on server. Reloading browser`) + window.location = pathname + } + } reachNavigate(to, options) clearTimeout(timeoutId) }) diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js index e1a77480ddc73..4f4b1efd363b4 100644 --- a/packages/gatsby/cache-dir/production-app.js +++ b/packages/gatsby/cache-dir/production-app.js @@ -18,6 +18,7 @@ import EnsureResources from "./ensure-resources" window.asyncRequires = asyncRequires window.___emitter = emitter window.___loader = loader +window.___webpackCompilationHash = window.webpackCompilationHash loader.addProdRequires(asyncRequires) setApiRunnerForLoader(apiRunner) diff --git a/packages/gatsby/cache-dir/static-entry.js b/packages/gatsby/cache-dir/static-entry.js index 0805e93fb6c63..4e0ddfa83622b 100644 --- a/packages/gatsby/cache-dir/static-entry.js +++ b/packages/gatsby/cache-dir/static-entry.js @@ -338,15 +338,17 @@ export default (pagePath, callback) => { } }) + const webpackCompilationHash = pageData.webpackCompilationHash + // Add page metadata for the current page - const windowPagePath = `/**/` + const windowPageData = `/**/` postBodyComponents.push(