diff --git a/examples/gatsbygram/package.json b/examples/gatsbygram/package.json index e3d85017ba7eb..a99b26e0d0733 100644 --- a/examples/gatsbygram/package.json +++ b/examples/gatsbygram/package.json @@ -6,7 +6,7 @@ "author": "Kyle Mathews ", "dependencies": { "core-js": "^2.5.5", - "gatsby": "^2.0.0", + "gatsby": "^2.3.4", "gatsby-image": "^2.0.5", "gatsby-plugin-glamor": "^2.0.5", "gatsby-plugin-google-analytics": "^2.0.5", diff --git a/packages/gatsby-plugin-guess-js/src/gatsby-ssr.js b/packages/gatsby-plugin-guess-js/src/gatsby-ssr.js index d0c0e96b1ddaf..8ec354f92c660 100644 --- a/packages/gatsby-plugin-guess-js/src/gatsby-ssr.js +++ b/packages/gatsby-plugin-guess-js/src/gatsby-ssr.js @@ -1,5 +1,4 @@ const _ = require(`lodash`) -const nodePath = require(`path`) const fs = require(`fs`) const React = require(`react`) @@ -12,18 +11,6 @@ function urlJoin(...parts) { }, ``) } -let pd = [] -const readPageData = () => { - if (pd.length > 0) { - return pd - } else { - pd = JSON.parse( - fs.readFileSync(nodePath.join(process.cwd(), `.cache`, `data.json`)) - ) - return pd - } -} - let s const readStats = () => { if (s) { @@ -37,19 +24,16 @@ const readStats = () => { } exports.onRenderBody = ( - { setHeadComponents, pathname, pathPrefix }, + { setHeadComponents, pathname, pathPrefix, loadPageDataSync }, pluginOptions ) => { if (process.env.NODE_ENV === `production`) { - const pagesData = readPageData() const stats = readStats() const matchedPaths = Object.keys( guess({ path: pathname, threshold: pluginOptions.minimumThreshold }) ) if (!_.isEmpty(matchedPaths)) { - const matchedPages = matchedPaths.map(match => - _.find(pagesData.pages, page => page.path === match) - ) + const matchedPages = matchedPaths.map(loadPageDataSync) let componentUrls = [] matchedPages.forEach(p => { if (p && p.componentChunkName) { diff --git a/packages/gatsby-plugin-offline/src/gatsby-browser.js b/packages/gatsby-plugin-offline/src/gatsby-browser.js index a9ee410438ff9..678ca55c8ea1f 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-browser.js +++ b/packages/gatsby-plugin-offline/src/gatsby-browser.js @@ -1,12 +1,9 @@ exports.registerServiceWorker = () => true -const prefetchedPathnames = [] +const prefetchedResources = [] const whitelistedPathnames = [] -exports.onServiceWorkerActive = ({ - getResourceURLsForPathname, - serviceWorker, -}) => { +exports.onServiceWorkerActive = ({ serviceWorker }) => { // if the SW has just updated then reset whitelisted paths and don't cache // stuff, since we're on the old revision until we navigate to another page if (window.___swUpdated) { @@ -26,15 +23,6 @@ exports.onServiceWorkerActive = ({ .call(nodes) .map(node => node.src || node.href || node.getAttribute(`data-href`)) - // Loop over all resources and fetch the page component and JSON - // to add it to the sw cache. - const prefetchedResources = [] - prefetchedPathnames.forEach(path => - getResourceURLsForPathname(path).forEach(resource => - prefetchedResources.push(resource) - ) - ) - const resources = [...headerResources, ...prefetchedResources] resources.forEach(resource => { // Create a prefetch link for each resource, so Workbox runtime-caches them @@ -69,12 +57,12 @@ function whitelistPathname(pathname, includesPrefix) { } } -exports.onPostPrefetchPathname = ({ pathname }) => { +exports.onPostPrefetch = ({ path, resourceUrls }) => { // do nothing if the SW has just updated, since we still have old pages in // memory which we don't want to be whitelisted if (window.___swUpdated) return - whitelistPathname(pathname, false) + whitelistPathname(path, false) // if SW is not installed, we need to record any prefetches // that happen so we can then add them to SW cache once installed @@ -85,6 +73,6 @@ exports.onPostPrefetchPathname = ({ pathname }) => { navigator.serviceWorker.controller.state === `activated` ) ) { - prefetchedPathnames.push(pathname) + Array.prototype.push.apply(prefetchedResources, resourceUrls) } } diff --git a/packages/gatsby/cache-dir/__tests__/find-page.js b/packages/gatsby/cache-dir/__tests__/find-page.js index c998b8e813c9a..41c6c12816313 100644 --- a/packages/gatsby/cache-dir/__tests__/find-page.js +++ b/packages/gatsby/cache-dir/__tests__/find-page.js @@ -8,23 +8,19 @@ describe(`find-page`, () => { { path: `/about/`, componentChunkName: `page-component---src-pages-test-js`, - jsonName: `about.json`, }, { path: `/about/me/`, componentChunkName: `page-component---src-pages-test-js`, - jsonName: `about-me.json`, }, { path: `/about/the best/`, componentChunkName: `page-component---src-pages-test-js`, - jsonName: `the-best.json`, }, { path: `/app/`, matchPath: `/app/*`, componentChunkName: `page-component---src-pages-app-js`, - jsonName: `app.json`, }, ] findPage = pageFinderFactory(newPages) @@ -58,7 +54,6 @@ describe(`find-page`, () => { { path: `/about/`, componentChunkName: `page-component---src-pages-test-js`, - jsonName: `about.json`, }, ] const findPage2 = pageFinderFactory(newPages, `/my-test-prefix`) diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js index 41edef94ab482..6dad7f633b4af 100644 --- a/packages/gatsby/cache-dir/__tests__/static-entry.js +++ b/packages/gatsby/cache-dir/__tests__/static-entry.js @@ -22,29 +22,6 @@ jest.mock( } ) -jest.mock( - `../data.json`, - () => { - return { - dataPaths: [ - { - [`about.json`]: `/400/about`, - }, - ], - pages: [ - { - path: `/about/`, - componentChunkName: `page-component---src-pages-test-js`, - jsonName: `about.json`, - }, - ], - } - }, - { - virtual: true, - } -) - const MOCK_FILE_INFO = { [`${process.cwd()}/public/webpack.stats.json`]: `{}`, [`${process.cwd()}/public/chunk-map.json`]: `{}`, diff --git a/packages/gatsby/cache-dir/api-runner-browser.js b/packages/gatsby/cache-dir/api-runner-browser.js index 0a4ff8b1168f4..f956b0494e13f 100644 --- a/packages/gatsby/cache-dir/api-runner-browser.js +++ b/packages/gatsby/cache-dir/api-runner-browser.js @@ -3,6 +3,8 @@ const { getResourcesForPathname, getResourcesForPathnameSync, getResourceURLsForPathname, + loadPage, + getPage, } = require(`./loader`).publicLoader exports.apiRunner = (api, args = {}, defaultReturn, argTransform) => { @@ -22,9 +24,14 @@ exports.apiRunner = (api, args = {}, defaultReturn, argTransform) => { return undefined } + // Deprecated April 2019. Use `getPage` instead args.getResourcesForPathnameSync = getResourcesForPathnameSync + // Deprecated April 2019. Use `loadPage` instead args.getResourcesForPathname = getResourcesForPathname + // Deprecated April 2019. Use resources passed in `onPostPrefetch` instead args.getResourceURLsForPathname = getResourceURLsForPathname + args.loadPage = loadPage + args.getPage = getPage const result = plugin.plugin[api](args, plugin.options) if (result && argTransform) { diff --git a/packages/gatsby/cache-dir/app.js b/packages/gatsby/cache-dir/app.js index a3da15eaa97f7..28acea0095f23 100644 --- a/packages/gatsby/cache-dir/app.js +++ b/packages/gatsby/cache-dir/app.js @@ -5,9 +5,9 @@ import domReady from "@mikaelkristiansson/domready" import socketIo from "./socketIo" import emitter from "./emitter" import { apiRunner, apiRunnerAsync } from "./api-runner-browser" -import loader, { setApiRunnerForLoader, postInitialRenderWork } from "./loader" +import loader, { setApiRunnerForLoader } from "./loader" +import devLoader from "./dev-loader" import syncRequires from "./sync-requires" -import pages from "./pages.json" window.___emitter = emitter setApiRunnerForLoader(apiRunner) @@ -46,19 +46,26 @@ apiRunnerAsync(`onClientEntry`).then(() => { ReactDOM.render )[0] - loader.addPagesArray(pages) loader.addDevRequires(syncRequires) - loader.getResourcesForPathname(window.location.pathname).then(() => { - const preferDefault = m => (m && m.default) || m - let Root = preferDefault(require(`./root`)) - domReady(() => { - renderer(, rootElement, () => { - postInitialRenderWork() - apiRunner(`onInitialClientRender`) + Promise.all([ + loader.loadPage(window.location.pathname), + loader.loadPage(`/dev-404-page/`), + loader.loadPage(`/404.html`).catch(err => null), + devLoader.loadPages(), + ]) + .then(() => { + const preferDefault = m => (m && m.default) || m + let Root = preferDefault(require(`./root`)) + domReady(() => { + renderer(, rootElement, () => { + apiRunner(`onInitialClientRender`) + }) }) }) - }) + .catch(err => { + console.log(err) + }) }) function supportsServiceWorkers(location, navigator) { diff --git a/packages/gatsby/cache-dir/dev-loader.js b/packages/gatsby/cache-dir/dev-loader.js new file mode 100644 index 0000000000000..93cdf3c4d4a3b --- /dev/null +++ b/packages/gatsby/cache-dir/dev-loader.js @@ -0,0 +1,49 @@ +// Initialized by calling loadPages +let pagesManifest = null + +const fetchPages = () => + new Promise((resolve, reject) => { + const req = new XMLHttpRequest() + req.open(`GET`, `/___pages`, true) + req.onreadystatechange = () => { + if (req.readyState == 4) { + if (req.status === 200) { + // TODO is this safe? Maybe just do this check in dev mode? + const contentType = req.getResponseHeader(`content-type`) + if (!contentType || !contentType.startsWith(`application/json`)) { + reject() + } else { + resolve(JSON.parse(req.responseText)) + } + } else { + reject() + } + } + } + req.send(null) + }) + +// Returns a promise that fetches the `/___pages` resource from the +// running `gatsby develop` server. It contains a map of all pages on +// the site (path -> page). Call `getPagesManifest()` to retrieve the +// pages. Used by dev-404-page to present a list of all pages +const loadPages = () => + fetchPages().then(pages => { + pagesManifest = pages + }) + +// Returns the map of all pages on the site (path -> page) +const getPagesManifest = () => { + if (pagesManifest === null) { + throw new Error( + `pages-manifest hasn't been initialized. Ensure the dev-loader/loadPages has been called first` + ) + } else { + return pagesManifest + } +} + +module.exports = { + loadPages, + getPagesManifest, +} diff --git a/packages/gatsby/cache-dir/ensure-resources.js b/packages/gatsby/cache-dir/ensure-resources.js index 0c37dd50333d6..23270d11ed8ea 100644 --- a/packages/gatsby/cache-dir/ensure-resources.js +++ b/packages/gatsby/cache-dir/ensure-resources.js @@ -17,7 +17,7 @@ class EnsureResources extends React.Component { this.state = { location: { ...location }, - pageResources: loader.getResourcesForPathnameSync(location.pathname), + pageResources: loader.getPageOr404(location.pathname), } } @@ -34,9 +34,7 @@ class EnsureResources extends React.Component { static getDerivedStateFromProps({ location }, prevState) { if (prevState.location !== location) { - const pageResources = loader.getResourcesForPathnameSync( - location.pathname - ) + const pageResources = loader.getPageOr404(location.pathname) return { pageResources, @@ -62,7 +60,7 @@ class EnsureResources extends React.Component { retryResources(nextProps) { const { pathname } = nextProps.location - if (!loader.getResourcesForPathnameSync(pathname)) { + if (!loader.getPage(pathname)) { // Store the previous and next location before resolving resources const prevLocation = this.props.location this.nextLocation = nextProps.location @@ -70,7 +68,7 @@ class EnsureResources extends React.Component { // Page resources won't be set in cases where the browser back button // or forward button is pushed as we can't wait as normal for resources // to load before changing the page. - loader.getResourcesForPathname(pathname).then(pageResources => { + loader.loadPage(pathname).then(pageResources => { // The page may have changed since we started this, in which case doesn't update if (this.nextLocation !== nextProps.location) { return diff --git a/packages/gatsby/cache-dir/find-page.js b/packages/gatsby/cache-dir/find-page.js deleted file mode 100644 index f25078f4f6b7b..0000000000000 --- a/packages/gatsby/cache-dir/find-page.js +++ /dev/null @@ -1,55 +0,0 @@ -// TODO add tests especially for handling prefixed links. -import { match as matchPath } from "@reach/router/lib/utils" -import stripPrefix from "./strip-prefix" - -const pageCache = {} - -export default (pages, pathPrefix = ``) => rawPathname => { - let pathname = decodeURIComponent(rawPathname) - - // Remove the pathPrefix from the pathname. - let trimmedPathname = stripPrefix(pathname, pathPrefix) - - // Remove any hashfragment - if (trimmedPathname.split(`#`).length > 1) { - trimmedPathname = trimmedPathname - .split(`#`) - .slice(0, -1) - .join(``) - } - - // Remove search query - if (trimmedPathname.split(`?`).length > 1) { - trimmedPathname = trimmedPathname - .split(`?`) - .slice(0, -1) - .join(``) - } - - if (pageCache[trimmedPathname]) { - return pageCache[trimmedPathname] - } - - let foundPage - // Array.prototype.find is not supported in IE so we use this somewhat odd - // work around. - pages.some(page => { - let pathToMatch = page.matchPath ? page.matchPath : page.path - if (matchPath(pathToMatch, trimmedPathname)) { - foundPage = page - pageCache[trimmedPathname] = page - return true - } - - // Finally, try and match request with default document. - if (matchPath(`${page.path}index.html`, trimmedPathname)) { - foundPage = page - pageCache[trimmedPathname] = page - return true - } - - return false - }) - - return foundPage -} diff --git a/packages/gatsby/cache-dir/json-store.js b/packages/gatsby/cache-dir/json-store.js index 632a5591aec20..3c476e94e2730 100644 --- a/packages/gatsby/cache-dir/json-store.js +++ b/packages/gatsby/cache-dir/json-store.js @@ -28,6 +28,7 @@ const getPathFromProps = props => class JSONStore extends React.Component { constructor(props) { super(props) + this.state = { staticQueryData: getStaticQueryData(), pageQueryData: getPageQueryData(), @@ -83,11 +84,10 @@ class JSONStore extends React.Component { render() { const data = this.state.pageQueryData[getPathFromProps(this.props)] // eslint-disable-next-line - const { pages, ...propsWithoutPages } = this.props + const { ...propsWithoutPages } = this.props if (!data) { return
} - return ( diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index 0d77a65805825..fe4d943192abc 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -1,166 +1,150 @@ -import pageFinderFactory from "./find-page" import emitter from "./emitter" import prefetchHelper from "./prefetch" +import { match } from "@reach/router/lib/utils" +import stripPrefix from "./strip-prefix" +// Generated during bootstrap +import matchPaths from "./match-paths.json" const preferDefault = m => (m && m.default) || m -let devGetPageData -let inInitialRender = true -let hasFetched = Object.create(null) +const pageNotFoundPaths = new Set() + +let apiRunner let syncRequires = {} let asyncRequires = {} -let jsonDataPaths = {} -let fetchHistory = [] -let fetchingPageResourceMapPromise = null -let fetchedPageResourceMap = false -/** - * Indicate if pages manifest is loaded - * - in production it is split to separate "pages-manifest" chunk that need to be lazy loaded, - * - in development it is part of single "common" chunk and is available from the start. - */ -let hasPageResourceMap = process.env.NODE_ENV !== `production` -let apiRunner -const failedPaths = {} -const MAX_HISTORY = 5 -const jsonPromiseStore = {} +const fetchedPageData = {} +const pageDatas = {} +const fetchPromiseStore = {} +let devGetPageData if (process.env.NODE_ENV !== `production`) { devGetPageData = require(`./socketIo`).getPageData } -/** - * Fetch resource map (pages data and paths to json files with results of - * queries) - */ -const fetchPageResourceMap = () => { - if (!fetchingPageResourceMapPromise) { - fetchingPageResourceMapPromise = new Promise(resolve => { - asyncRequires - .data() - .then(({ pages, dataPaths }) => { - // TODO — expose proper way to access this data from plugins. - // Need to come up with an API for plugins to access - // site info. - window.___dataPaths = dataPaths - queue.addPagesArray(pages) - queue.addDataPaths(dataPaths) - hasPageResourceMap = true - resolve((fetchedPageResourceMap = true)) - }) - .catch(e => { - console.warn( - `Failed to fetch pages manifest. Gatsby will reload on next navigation.` - ) - // failed to grab pages metadata - // for now let's just resolve this - on navigation this will cause missing resources - // and will trigger page reload and then it will retry - // this can happen with service worker updates when webpack manifest points to old - // chunk that no longer exists on server - resolve((fetchedPageResourceMap = true)) - }) - }) +// Cache for `cleanAndFindPath()`. In case `match-paths.json` is large +const cleanAndFindPathCache = {} + +// Given a raw URL path, returns the cleaned version of it (trim off +// `#` and query params), or if it matches an entry in +// `match-paths.json`, its matched path is returned +// +// E.g `/foo?bar=far` => `/foo` +// +// Or if `match-paths.json` contains `{ "/foo*": "/page1", ...}`, then +// `/foo?bar=far` => `/page1` +const cleanAndFindPath = rawPathname => { + let pathname = decodeURIComponent(rawPathname) + // Remove the pathPrefix from the pathname. + let trimmedPathname = stripPrefix(pathname, __PATH_PREFIX__) + // Remove any hashfragment + if (trimmedPathname.split(`#`).length > 1) { + trimmedPathname = trimmedPathname + .split(`#`) + .slice(0, -1) + .join(``) } - return fetchingPageResourceMapPromise -} -const createJsonURL = jsonName => `${__PATH_PREFIX__}/static/d/${jsonName}.json` -const createComponentUrls = componentChunkName => - window.___chunkMapping[componentChunkName].map( - chunk => __PATH_PREFIX__ + chunk - ) - -const fetchResource = resourceName => { - // Find resource - let resourceFunction - if (resourceName.slice(0, 12) === `component---`) { - resourceFunction = asyncRequires.components[resourceName] - } else { - if (resourceName in jsonPromiseStore) { - resourceFunction = () => jsonPromiseStore[resourceName] - } else { - resourceFunction = () => { - const fetchPromise = new Promise((resolve, reject) => { - const url = createJsonURL(jsonDataPaths[resourceName]) - const req = new XMLHttpRequest() - req.open(`GET`, url, true) - req.withCredentials = true - req.onreadystatechange = () => { - if (req.readyState == 4) { - if (req.status === 200) { - resolve(JSON.parse(req.responseText)) - } else { - delete jsonPromiseStore[resourceName] - reject() - } - } - } - req.send(null) - }) - jsonPromiseStore[resourceName] = fetchPromise - return fetchPromise - } - } + // Remove search query + if (trimmedPathname.split(`?`).length > 1) { + trimmedPathname = trimmedPathname + .split(`?`) + .slice(0, -1) + .join(``) + } + if (cleanAndFindPathCache[trimmedPathname]) { + return cleanAndFindPathCache[trimmedPathname] } - // Download the resource - hasFetched[resourceName] = true - return new Promise(resolve => { - const fetchPromise = resourceFunction() - let failed = false - return fetchPromise - .catch(() => { - failed = true - }) - .then(component => { - fetchHistory.push({ - resource: resourceName, - succeeded: !failed, - }) - - fetchHistory = fetchHistory.slice(-MAX_HISTORY) - - resolve(component) - }) + let foundPath + Object.keys(matchPaths).some(matchPath => { + if (match(matchPath, trimmedPathname)) { + foundPath = matchPaths[matchPath] + return foundPath + } + // Finally, try and match request with default document. + if (trimmedPathname === `/index.html`) { + foundPath = `/` + return foundPath + } + return false }) + if (!foundPath) { + foundPath = trimmedPathname + } + cleanAndFindPathCache[trimmedPathname] = foundPath + return foundPath } -const prefetchResource = resourceName => { - if (resourceName.slice(0, 12) === `component---`) { - return Promise.all( - createComponentUrls(resourceName).map(url => prefetchHelper(url)) - ) +const cachedFetch = (resourceName, fetchFn) => { + if (resourceName in fetchPromiseStore) { + return fetchPromiseStore[resourceName] } else { - const url = createJsonURL(jsonDataPaths[resourceName]) - return prefetchHelper(url) + const promise = fetchFn(resourceName) + fetchPromiseStore[resourceName] = promise + return promise.catch(err => { + delete fetchPromiseStore[resourceName] + return err + }) } } -const getResourceModule = resourceName => - fetchResource(resourceName).then(preferDefault) +const doFetch = url => + new Promise((resolve, reject) => { + const req = new XMLHttpRequest() + req.open(`GET`, url, true) + req.withCredentials = true + req.onreadystatechange = () => { + if (req.readyState == 4) { + resolve(req) + } + } + req.send(null) + }) -const appearsOnLine = () => { - const isOnLine = navigator.onLine - if (typeof isOnLine === `boolean`) { - return isOnLine +const handlePageDataResponse = (path, req) => { + fetchedPageData[path] = true + if (req.status === 200) { + const contentType = req.getResponseHeader(`content-type`) + if (!contentType || !contentType.startsWith(`application/json`)) { + pageNotFoundPaths.add(path) + return null + } else { + const pageData = JSON.parse(req.responseText) + pageDatas[path] = pageData + return pageData + } + } else if (req.status === 404) { + pageNotFoundPaths.add(path) + return null + } else { + throw new Error(`error fetching page`) } +} - // If no navigator.onLine support assume onLine if any of last N fetches succeeded - const succeededFetch = fetchHistory.find(entry => entry.succeeded) - return !!succeededFetch +const fetchPageData = path => { + const url = makePageDataUrl(path) + return cachedFetch(url, doFetch).then(req => + handlePageDataResponse(path, req) + ) } -const handleResourceLoadError = (path, message) => { - if (!failedPaths[path]) { - failedPaths[path] = message - } +const createComponentUrls = componentChunkName => + window.___chunkMapping[componentChunkName].map( + chunk => __PATH_PREFIX__ + chunk + ) - if ( - appearsOnLine() && - window.location.pathname.replace(/\/$/g, ``) !== path.replace(/\/$/g, ``) - ) { - window.location.pathname = path - } +const fetchComponent = chunkName => asyncRequires.components[chunkName]() + +const stripSurroundingSlashes = s => { + s = s[0] === `/` ? s.slice(1) : s + s = s.endsWith(`/`) ? s.slice(0, -1) : s + return s +} + +const makePageDataUrl = path => { + const fixedPath = path === `/` ? `index` : stripSurroundingSlashes(path) + return `${__PATH_PREFIX__}/page-data/${fixedPath}/page-data.json` } const onPrefetchPathname = pathname => { @@ -170,40 +154,27 @@ const onPrefetchPathname = pathname => { } } -const onPostPrefetchPathname = pathname => { - if (!prefetchCompleted[pathname]) { - apiRunner(`onPostPrefetchPathname`, { pathname }) - prefetchCompleted[pathname] = true - } -} - -/** - * Check if we should fallback to resources for 404 page if resources for a page are not found - * - * We can't do that when we don't have full pages manifest - we don't know if page exist or not if we don't have it. - * We also can't do that on initial render / mount in case we just can't load resources needed for first page. - * Not falling back to 404 resources will cause "EnsureResources" component to handle scenarios like this with - * potential reload - * @param {string} path Path to a page - */ -const shouldFallbackTo404Resources = path => - (hasPageResourceMap || inInitialRender) && path !== `/404.html` - // Note we're not actively using the path data atm. There // could be future optimizations however around trying to ensure // we load all resources for likely-to-be-visited paths. // let pathArray = [] // let pathCount = {} -let findPage let pathScriptsCache = {} let prefetchTriggered = {} let prefetchCompleted = {} let disableCorePrefetching = false +const onPostPrefetch = url => { + if (!prefetchCompleted[url]) { + apiRunner(`onPostPrefetch`, { url }) + prefetchCompleted[url] = true + } +} + const queue = { - addPagesArray: newPages => { - findPage = pageFinderFactory(newPages, __PATH_PREFIX__) + addPageData: pageData => { + pageDatas[pageData.path] = pageData }, addDevRequires: devRequires => { syncRequires = devRequires @@ -211,16 +182,11 @@ const queue = { addProdRequires: prodRequires => { asyncRequires = prodRequires }, - addDataPaths: dataPaths => { - jsonDataPaths = dataPaths - }, // Hovering on a link is a very strong indication the user is going to // click on it soon so let's start prefetching resources for this // pathname. - hovering: path => { - queue.getResourcesForPathname(path) - }, - enqueue: path => { + hovering: path => queue.loadPage(path), + enqueue: rawPath => { if (!apiRunner) console.error(`Run setApiRunnerForLoader() before enqueing paths`) @@ -236,7 +202,7 @@ const queue = { // Tell plugins with custom prefetching logic that they should start // prefetching this path. - onPrefetchPathname(path) + onPrefetchPathname(rawPath) // If a plugin has disabled core prefetching, stop now. if (disableCorePrefetching.some(a => a)) { @@ -244,204 +210,164 @@ const queue = { } // Check if the page exists. - let page = findPage(path) + let realPath = cleanAndFindPath(rawPath) - // In production, we lazy load page metadata. If that - // hasn't been fetched yet, start fetching it now. - if ( - process.env.NODE_ENV === `production` && - !page && - !fetchedPageResourceMap - ) { - // If page wasn't found check and we didn't fetch resources map for - // all pages, wait for fetch to complete and try find page again - return fetchPageResourceMap().then(() => queue.enqueue(path)) - } - - if (!page) { - return false + if (pageDatas[realPath]) { + return true } if ( process.env.NODE_ENV !== `production` && process.env.NODE_ENV !== `test` ) { - devGetPageData(page.path) + // Ensure latest version of page data is in the JSON store + devGetPageData(realPath) } - // Prefetch resources. if (process.env.NODE_ENV === `production`) { - Promise.all([ - prefetchResource(page.jsonName), - prefetchResource(page.componentChunkName), - ]).then(() => { - // Tell plugins the path has been successfully prefetched - onPostPrefetchPathname(path) - }) + const pageDataUrl = makePageDataUrl(realPath) + prefetchHelper(pageDataUrl) + .then(() => + // This was just prefetched, so will return a response from + // the cache instead of making another request to the server + fetchPageData(realPath) + ) + .then(pageData => { + // Tell plugins the path has been successfully prefetched + const chunkName = pageData.componentChunkName + const componentUrls = createComponentUrls(chunkName) + return Promise.all(componentUrls.map(prefetchHelper)).then(() => { + const resourceUrls = [pageDataUrl].concat(componentUrls) + onPostPrefetch({ + path: rawPath, + resourceUrls, + }) + }) + }) } return true }, - getPage: pathname => findPage(pathname), - - getResourceURLsForPathname: path => { - const page = findPage(path) - if (page) { - return [ - ...createComponentUrls(page.componentChunkName), - createJsonURL(jsonDataPaths[page.jsonName]), - ] - } else { - return null - } - }, - - getResourcesForPathnameSync: path => { - const page = findPage(path) - if (page) { - return pathScriptsCache[page.path] - } else if (shouldFallbackTo404Resources(path)) { - return queue.getResourcesForPathnameSync(`/404.html`) - } else { - return null - } - }, + isPageNotFound: pathname => pageNotFoundPaths.has(pathname), - // Get resources (code/data) for a path. Fetches metdata first - // if necessary and then the code/data bundles. Used for prefetching - // and getting resources for page changes. - getResourcesForPathname: path => + loadPageData: rawPath => new Promise((resolve, reject) => { - // Production code path - if (failedPaths[path]) { - handleResourceLoadError( - path, - `Previously detected load failure for "${path}"` - ) - reject() - return - } - const page = findPage(path) - - // In production, we lazy load page metadata. If that - // hasn't been fetched yet, start fetching it now. - if ( - !page && - !fetchedPageResourceMap && - process.env.NODE_ENV === `production` - ) { - // If page wasn't found check and we didn't fetch resources map for - // all pages, wait for fetch to complete and try to get resources again - fetchPageResourceMap().then(() => - resolve(queue.getResourcesForPathname(path)) - ) - return - } - - if (!page) { - if (shouldFallbackTo404Resources(path)) { - console.log(`A page wasn't found for "${path}"`) - - // Preload the custom 404 page - resolve(queue.getResourcesForPathname(`/404.html`)) - return - } - - resolve() - return - } - - // Use the path from the page so the pathScriptsCache uses - // the normalized path. - path = page.path - - // Check if it's in the cache already. - if (pathScriptsCache[path]) { - emitter.emit(`onPostLoadPageResources`, { - page, - pageResources: pathScriptsCache[path], + const realPath = cleanAndFindPath(rawPath) + if (!fetchedPageData[realPath]) { + fetchPageData(realPath).then(pageData => { + if (process.env.NODE_ENV !== `production`) { + devGetPageData(realPath) + } + resolve(queue.loadPageData(rawPath)) }) - resolve(pathScriptsCache[path]) - return + } else { + if (pageDatas[realPath]) { + resolve(pageDatas[realPath]) + } else { + reject(new Error(`page not found`)) + } } + }), - // Nope, we need to load resource(s) - emitter.emit(`onPreLoadPageResources`, { - path, + loadPage: rawPath => + queue + .loadPageData(rawPath) + .then(pageData => { + if (process.env.NODE_ENV !== `production`) { + const component = syncRequires.components[pageData.componentChunkName] + return [pageData, component] + } else { + return cachedFetch(pageData.componentChunkName, fetchComponent) + .then(preferDefault) + .then(component => [pageData, component]) + } }) + .then(([pageData, component]) => { + const page = { + componentChunkName: pageData.componentChunkName, + path: pageData.path, + compilationHash: pageData.compilationHash, + } + + const jsonData = { + data: pageData.data, + pageContext: pageData.pageContext, + } - // In development we know the code is loaded already - // so we just return with it immediately. - if (process.env.NODE_ENV !== `production`) { const pageResources = { - component: syncRequires.components[page.componentChunkName], + component, + json: jsonData, page, } - // Add to the cache. - pathScriptsCache[path] = pageResources - devGetPageData(page.path).then(pageData => { - emitter.emit(`onPostLoadPageResources`, { - page, - pageResources, + pathScriptsCache[cleanAndFindPath(rawPath)] = pageResources + emitter.emit(`onPostLoadPageResources`, { + page: pageResources, + pageResources, + }) + if (process.env.NODE_ENV === `production`) { + const pageDataUrl = makePageDataUrl(cleanAndFindPath(rawPath)) + const componentUrls = createComponentUrls(pageData.componentChunkName) + const resourceUrls = [pageDataUrl].concat(componentUrls) + onPostPrefetch({ + path: rawPath, + resourceUrls, }) - // Tell plugins the path has been successfully prefetched - onPostPrefetchPathname(path) + } - resolve(pageResources) - }) - } else { - Promise.all([ - getResourceModule(page.componentChunkName), - getResourceModule(page.jsonName), - ]).then(([component, json]) => { - if (!(component && json)) { - resolve(null) - return - } + return pageResources + }) + .catch(err => null), - const pageResources = { - component, - json, - page, - } - pageResources.page.jsonURL = createJsonURL( - jsonDataPaths[page.jsonName] - ) - pathScriptsCache[path] = pageResources - resolve(pageResources) - - emitter.emit(`onPostLoadPageResources`, { - page, - pageResources, - }) + getPage: rawPath => pathScriptsCache[cleanAndFindPath(rawPath)], - // Tell plugins the path has been successfully prefetched - onPostPrefetchPathname(path) - }) - } - }), -} + getPageOr404: rawPath => { + const page = queue.getPage(rawPath) + if (page) { + return page + } else if (rawPath !== `/404.html`) { + return queue.getPage(`/404.html`) + } else { + return null + } + }, -export const postInitialRenderWork = () => { - inInitialRender = false - if (process.env.NODE_ENV === `production`) { - // We got all resources needed for first mount, - // we can fetch resources for all pages. - fetchPageResourceMap() - } + getResourceURLsForPathname: path => { + const pageData = queue.getPage(path) + if (pageData) { + // Original implementation also concatenated the jsonDataPath + // for the page + return createComponentUrls(pageData.componentChunkName) + } else { + return null + } + }, } +// Deprecated April 2019. Used to fetch the pages-manifest. Now it's a +// noop +export const postInitialRenderWork = () => {} + export const setApiRunnerForLoader = runner => { apiRunner = runner disableCorePrefetching = apiRunner(`disableCorePrefetching`) } export const publicLoader = { - getResourcesForPathname: queue.getResourcesForPathname, + // Deprecated April 2019. Use `loadPage` instead + getResourcesForPathname: queue.loadPage, + // Deprecated April 2019. Use `getPage` instead + getResourcesForPathnameSync: queue.getPage, + // Deprecated April 2019. Query results used to be in a separate + // file, but are now included in the page-data.json, which is + // already loaded into the browser by the time this function is + // called. Use the resource URLs passed in `onPostPrefetch` instead. getResourceURLsForPathname: queue.getResourceURLsForPathname, - getResourcesForPathnameSync: queue.getResourcesForPathnameSync, + + loadPage: queue.loadPage, + getPage: queue.getPage, + getPageOr404: queue.getPageOr404, } export default queue diff --git a/packages/gatsby/cache-dir/navigation.js b/packages/gatsby/cache-dir/navigation.js index 6289cc2d3f8a1..5acdbef4d6700 100644 --- a/packages/gatsby/cache-dir/navigation.js +++ b/packages/gatsby/cache-dir/navigation.js @@ -18,7 +18,7 @@ function maybeRedirect(pathname) { if (redirect != null) { if (process.env.NODE_ENV !== `production`) { - const pageResources = loader.getResourcesForPathnameSync(pathname) + const pageResources = loader.getPage(pathname) if (pageResources != null) { console.error( @@ -81,7 +81,19 @@ const navigate = (to, options = {}) => { }) }, 1000) - loader.getResourcesForPathname(pathname).then(pageResources => { + loader.loadPage(pathname).then(pageResources => { + // If the loaded page has a different compilation hash to the + // window, then a rebuild has occurred on the server. Reload. + if (process.env.NODE_ENV === `production` && pageResources) { + if (pageResources.page.compilationHash !== window.___compilationHash) { + console.log( + `compilation has different. window = ${ + window.___compilationHash + }, page = ${pageResources.page.compilationHash}` + ) + window.location = pathname + } + } reachNavigate(to, options) clearTimeout(timeoutId) }) diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js index c3067b678e9d5..0b8d06ced21d1 100644 --- a/packages/gatsby/cache-dir/production-app.js +++ b/packages/gatsby/cache-dir/production-app.js @@ -2,7 +2,6 @@ import { apiRunner, apiRunnerAsync } from "./api-runner-browser" import React, { createElement } from "react" import ReactDOM from "react-dom" import { Router, navigate } from "@reach/router" -import { match } from "@reach/router/lib/utils" import { ScrollContext } from "gatsby-react-router-scroll" import domReady from "@mikaelkristiansson/domready" import { @@ -13,15 +12,15 @@ import { import emitter from "./emitter" import PageRenderer from "./page-renderer" import asyncRequires from "./async-requires" -import loader, { setApiRunnerForLoader, postInitialRenderWork } from "./loader" +import loader, { setApiRunnerForLoader } from "./loader" import EnsureResources from "./ensure-resources" window.asyncRequires = asyncRequires window.___emitter = emitter window.___loader = loader +window.___compilationHash = window.pageData.compilationHash -loader.addPagesArray([window.page]) -loader.addDataPaths({ [window.page.jsonName]: window.dataPath }) +loader.addPageData([window.pageData]) loader.addProdRequires(asyncRequires) setApiRunnerForLoader(apiRunner) @@ -61,29 +60,26 @@ apiRunnerAsync(`onClientEntry`).then(() => { } } - const { page, location: browserLoc } = window + const { pageData, location: browserLoc } = window if ( // Make sure the window.page object is defined - page && + pageData && // The canonical path doesn't match the actual path (i.e. the address bar) - __PATH_PREFIX__ + page.path !== browserLoc.pathname && - // ...and if matchPage is specified, it also doesn't match the actual path - (!page.matchPath || - !match(__PATH_PREFIX__ + page.matchPath, browserLoc.pathname)) && + __PATH_PREFIX__ + pageData.path !== browserLoc.pathname && // Ignore 404 pages, since we want to keep the same URL - page.path !== `/404.html` && - !page.path.match(/^\/404\/?$/) && + pageData.path !== `/404.html` && + !pageData.path.match(/^\/404\/?$/) && // Also ignore the offline shell (since when using the offline plugin, all // pages have this canonical path) - !page.path.match(/^\/offline-plugin-app-shell-fallback\/?$/) + !pageData.path.match(/^\/offline-plugin-app-shell-fallback\/?$/) ) { navigate( - __PATH_PREFIX__ + page.path + browserLoc.search + browserLoc.hash, + __PATH_PREFIX__ + pageData.path + browserLoc.search + browserLoc.hash, { replace: true } ) } - loader.getResourcesForPathname(browserLoc.pathname).then(() => { + loader.loadPage(browserLoc.pathname).then(() => { const Root = () => createElement( Router, @@ -117,7 +113,6 @@ apiRunnerAsync(`onClientEntry`).then(() => { ? document.getElementById(`___gatsby`) : void 0, () => { - postInitialRenderWork() apiRunner(`onInitialClientRender`) } ) diff --git a/packages/gatsby/cache-dir/public-page-renderer-dev.js b/packages/gatsby/cache-dir/public-page-renderer-dev.js index 5d283d011eaf9..b274fd737386e 100644 --- a/packages/gatsby/cache-dir/public-page-renderer-dev.js +++ b/packages/gatsby/cache-dir/public-page-renderer-dev.js @@ -6,7 +6,7 @@ import loader from "./loader" import JSONStore from "./json-store" const DevPageRenderer = ({ location }) => { - const pageResources = loader.getResourcesForPathnameSync(location.pathname) + const pageResources = loader.getPage(location.pathname) return React.createElement(JSONStore, { pages, location, diff --git a/packages/gatsby/cache-dir/public-page-renderer-prod.js b/packages/gatsby/cache-dir/public-page-renderer-prod.js index 9734692daca9d..dc814f82488ab 100644 --- a/packages/gatsby/cache-dir/public-page-renderer-prod.js +++ b/packages/gatsby/cache-dir/public-page-renderer-prod.js @@ -5,7 +5,7 @@ import InternalPageRenderer from "./page-renderer" import loader from "./loader" const ProdPageRenderer = ({ location }) => { - const pageResources = loader.getResourcesForPathnameSync(location.pathname) + const pageResources = loader.getPageOr404(location.pathname) return React.createElement(InternalPageRenderer, { location, pageResources, diff --git a/packages/gatsby/cache-dir/root.js b/packages/gatsby/cache-dir/root.js index e4c0147b03fd6..247e4307102b8 100644 --- a/packages/gatsby/cache-dir/root.js +++ b/packages/gatsby/cache-dir/root.js @@ -8,9 +8,8 @@ import { RouteUpdates, } from "./navigation" import { apiRunner } from "./api-runner-browser" -import syncRequires from "./sync-requires" -import pages from "./pages.json" import loader from "./loader" +import devLoader from "./dev-loader" import JSONStore from "./json-store" import EnsureResources from "./ensure-resources" @@ -38,12 +37,10 @@ navigationInit() class RouteHandler extends React.Component { render() { let { location } = this.props + const pages = devLoader.getPagesManifest() + const pagePaths = Object.keys(pages) - // check if page exists - in dev pages are sync loaded, it's safe to use - // loader.getPage - let page = loader.getPage(location.pathname) - - if (page) { + if (!loader.isPageNotFound(location.pathname)) { return ( {locationAndPageResources => ( @@ -52,24 +49,20 @@ class RouteHandler extends React.Component { location={location} shouldUpdateScroll={shouldUpdateScroll} > - + )} ) } else { - const dev404Page = pages.find(p => /^\/dev-404-page\/?$/.test(p.path)) - const Dev404Page = syncRequires.components[dev404Page.componentChunkName] + const dev404Page = loader.getPage(`/dev-404-page/`) + const Dev404Page = dev404Page.component if (!loader.getPage(`/404.html`)) { return ( - + ) } @@ -79,13 +72,9 @@ class RouteHandler extends React.Component { {locationAndPageResources => ( + } {...this.props} /> diff --git a/packages/gatsby/cache-dir/socketIo.js b/packages/gatsby/cache-dir/socketIo.js index 978f7db8966d1..1548f39d23d98 100644 --- a/packages/gatsby/cache-dir/socketIo.js +++ b/packages/gatsby/cache-dir/socketIo.js @@ -20,7 +20,7 @@ export default function socketIo() { const didDataChange = (msg, queryData) => !(msg.payload.id in queryData) || - JSON.stringify(msg.payload.result) !== + JSON.stringify(msg.payload) !== JSON.stringify(queryData[msg.payload.id]) socket.on(`message`, msg => { @@ -28,14 +28,14 @@ export default function socketIo() { if (didDataChange(msg, staticQueryData)) { staticQueryData = { ...staticQueryData, - [msg.payload.id]: msg.payload.result, + [msg.payload.id]: msg.payload, } } } else if (msg.type === `pageQueryResult`) { if (didDataChange(msg, pageQueryData)) { pageQueryData = { ...pageQueryData, - [msg.payload.id]: msg.payload.result, + [msg.payload.id]: msg.payload, } } } else if (msg.type === `overlayError`) { diff --git a/packages/gatsby/cache-dir/static-entry.js b/packages/gatsby/cache-dir/static-entry.js index aa3b9cfb95daa..6f924175c9082 100644 --- a/packages/gatsby/cache-dir/static-entry.js +++ b/packages/gatsby/cache-dir/static-entry.js @@ -7,13 +7,8 @@ const { get, merge, isObject, flatten, uniqBy } = require(`lodash`) const apiRunner = require(`./api-runner-ssr`) const syncRequires = require(`./sync-requires`) -const { dataPaths, pages } = require(`./data.json`) const { version: gatsbyVersion } = require(`gatsby/package.json`) -// Speed up looking up pages. -const pagesObjectMap = new Map() -pages.forEach(p => pagesObjectMap.set(p.path, p)) - const stats = JSON.parse( fs.readFileSync(`${process.cwd()}/public/webpack.stats.json`, `utf-8`) ) @@ -45,7 +40,27 @@ try { Html = Html && Html.__esModule ? Html.default : Html -const getPage = path => pagesObjectMap.get(path) +const getPageDataPath = path => { + const fixedPagePath = path === `/` ? `index` : path + return join(`page-data`, fixedPagePath, `page-data.json`) +} + +const getPageDataUrl = pagePath => { + const pageDataPath = getPageDataPath(pagePath) + return `${__PATH_PREFIX__}/${pageDataPath}` +} + +const getPageDataFile = pagePath => { + const pageDataPath = getPageDataPath(pagePath) + return join(process.cwd(), `public`, pageDataPath) +} + +const loadPageDataSync = pagePath => { + const pageDataPath = getPageDataPath(pagePath) + const pageDataFile = join(process.cwd(), `public`, pageDataPath) + const pageDataJson = fs.readFileSync(pageDataFile) + return JSON.parse(pageDataJson) +} const createElement = React.createElement @@ -122,33 +137,20 @@ export default (pagePath, callback) => { postBodyComponents = sanitizeComponents(components) } - const page = getPage(pagePath) - - let dataAndContext = {} - if (page.jsonName in dataPaths) { - const pathToJsonData = join( - process.cwd(), - `/public/static/d`, - `${dataPaths[page.jsonName]}.json` - ) - try { - dataAndContext = JSON.parse(fs.readFileSync(pathToJsonData)) - } catch (e) { - console.log(`error`, pathToJsonData, e) - process.exit() - } - } + const pageDataRaw = fs.readFileSync(getPageDataFile(pagePath)) + const pageData = JSON.parse(pageDataRaw) + const pageDataUrl = getPageDataUrl(pagePath) + const { componentChunkName } = pageData class RouteHandler extends React.Component { render() { const props = { ...this.props, - ...dataAndContext, - pathContext: dataAndContext.pageContext, + ...pageData, } const pageElement = createElement( - syncRequires.components[page.componentChunkName], + syncRequires.components[componentChunkName], props ) @@ -212,7 +214,7 @@ export default (pagePath, callback) => { // Create paths to scripts let scriptsAndStyles = flatten( - [`app`, page.componentChunkName].map(s => { + [`app`, componentChunkName].map(s => { const fetchKey = `assetsByChunkName[${s}]` let chunks = get(stats, fetchKey) @@ -266,6 +268,7 @@ export default (pagePath, callback) => { setPostBodyComponents, setBodyProps, pathname: pagePath, + loadPageDataSync, bodyHtml, scripts, styles, @@ -287,16 +290,13 @@ export default (pagePath, callback) => { ) }) - if (page.jsonName in dataPaths) { - const dataPath = `${__PATH_PREFIX__}/static/d/${ - dataPaths[page.jsonName] - }.json` + if (pageData) { headComponents.push( ) @@ -334,11 +334,7 @@ export default (pagePath, callback) => { }) // Add page metadata for the current page - const windowData = `/**/` + const windowData = `/**/` postBodyComponents.push(