Implement "on demand entries"

client/on-demand-entries-client.js

@@ -0,0 +1,36 @@
+/* global location */
+
+import Router from '../lib/router'
+import fetch from 'unfetch'
+
+// Ping on every page change
+const originalOnRouteChangeComplete = Router.onRouteChangeComplete
+Router.onRouteChangeComplete = function (...args) {
+  ping()
+  if (originalOnRouteChangeComplete) originalOnRouteChangeComplete(...args)
+}
+
+async function ping () {
+  try {
+    const url = `/on-demand-entries-ping?page=${Router.pathname}`
+    const res = await fetch(url)
+    const payload = await res.json()
+    if (payload.invalid) {
+      location.reload()
+    }
+  } catch (err) {
+    console.error(`Error with on-demand-entries-ping: ${err.message}`)
+  }
+}
+
+async function runPinger () {
+  while (true) {
+    await new Promise((resolve) => setTimeout(resolve, 5000))
+    await ping()
+  }
+}
+
+runPinger()
+  .catch((err) => {
+    console.error(err)
+  })

lib/router/router.js

@@ -224,6 +224,10 @@ export default class Router extends EventEmitter {
   }
 
   async prefetch (url) {
+    // We don't add support for prefetch in the development mode.
+    // If we do that, our on-demand-entries optimization won't performs better
+    if (process.env.NODE_ENV === 'development') return
+
     const { pathname } = parse(url)
     const route = toRoute(pathname)
     return this.prefetchQueue.add(() => this.fetchRoute(route))

package.json

@@ -78,6 +78,7 @@
     "source-map-support": "0.4.11",
     "strip-ansi": "3.0.1",
     "styled-jsx": "0.5.7",
+    "touch": "1.0.0",
     "unfetch": "2.1.0",
     "url": "0.11.0",
     "uuid": "3.0.1",

server/build/webpack.js

@@ -6,7 +6,6 @@ import WriteFilePlugin from 'write-file-webpack-plugin'
 import FriendlyErrorsWebpackPlugin from 'friendly-errors-webpack-plugin'
 import CaseSensitivePathPlugin from 'case-sensitive-paths-webpack-plugin'
 import UnlinkFilePlugin from './plugins/unlink-file-plugin'
-import WatchPagesPlugin from './plugins/watch-pages-plugin'
 import JsonPagesPlugin from './plugins/json-pages-plugin'
 import getConfig from '../config'
 import * as babelCore from 'babel-core'
@@ -40,8 +39,18 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
     const entries = { 'main.js': mainJS }
 
     const pages = await glob('pages/**/*.js', { cwd: dir })
-    for (const p of pages) {
-      entries[join('bundles', p)] = [...defaultEntries, `./${p}?entry`]
+    const devPages = pages.filter((p) => p === 'pages/_document.js' || p === 'pages/_error.js')
+
+    // In the dev environment, on-demand-entry-handler will take care of
+    // managing pages.
+    if (dev) {
+      for (const p of devPages) {
+        entries[join('bundles', p)] = [...defaultEntries, `./${p}?entry`]
+      }
+    } else {
+      for (const p of pages) {
+        entries[join('bundles', p)] = [...defaultEntries, `./${p}?entry`]
+      }
     }
 
     for (const p of defaultPages) {
@@ -81,6 +90,9 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
         return count >= minChunks
       }
     }),
+    new webpack.DefinePlugin({
+      'process.env.NODE_ENV': JSON.stringify(dev ? 'development' : 'production')
+    }),
     new JsonPagesPlugin(),
     new CaseSensitivePathPlugin()
   ]
@@ -89,17 +101,13 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
     plugins.push(
       new webpack.HotModuleReplacementPlugin(),
       new webpack.NoEmitOnErrorsPlugin(),
-      new UnlinkFilePlugin(),
-      new WatchPagesPlugin(dir)
+      new UnlinkFilePlugin()
     )
     if (!quiet) {
       plugins.push(new FriendlyErrorsWebpackPlugin())
     }
   } else {
     plugins.push(
-      new webpack.DefinePlugin({
-        'process.env.NODE_ENV': JSON.stringify('production')
-      }),
       new webpack.optimize.UglifyJsPlugin({
         compress: { warnings: false },
         sourceMap: false

server/hot-reloader.js

@@ -1,10 +1,12 @@
 import { join, relative, sep } from 'path'
 import webpackDevMiddleware from 'webpack-dev-middleware'
 import webpackHotMiddleware from 'webpack-hot-middleware'
+import onDemandEntryHandler from './on-demand-entry-handler'
 import isWindowsBash from 'is-windows-bash'
 import webpack from './build/webpack'
 import clean from './build/clean'
 import readPage from './read-page'
+import getConfig from './config'
 
 export default class HotReloader {
   constructor (dir, { quiet } = {}) {
@@ -20,6 +22,8 @@ export default class HotReloader {
     this.prevChunkNames = null
     this.prevFailedChunkNames = null
     this.prevChunkHashes = null
+
+    this.config = getConfig(dir)
   }
 
   async run (req, res) {
@@ -145,10 +149,16 @@ export default class HotReloader {
     })
 
     this.webpackHotMiddleware = webpackHotMiddleware(compiler, { log: false })
+    this.onDemandEntries = onDemandEntryHandler(this.webpackDevMiddleware, compiler, {
+      dir: this.dir,
+      dev: true,
+      ...this.config.onDemandEntries
+    })
 
     this.middlewares = [
       this.webpackDevMiddleware,
-      this.webpackHotMiddleware
+      this.webpackHotMiddleware,
+      this.onDemandEntries.middleware()
     ]
   }
 
@@ -184,6 +194,10 @@ export default class HotReloader {
   send (action, ...args) {
     this.webpackHotMiddleware.publish({ action, data: args })
   }
+
+  ensurePage (page) {
+    return this.onDemandEntries.ensurePage(page)
+  }
 }
 
 function deleteCache (path) {

server/index.js

@@ -22,9 +22,9 @@ export default class Server {
     this.dir = resolve(dir)
     this.dev = dev
     this.quiet = quiet
-    this.renderOpts = { dir: this.dir, dev, staticMarkup }
     this.router = new Router()
     this.hotReloader = dev ? new HotReloader(this.dir, { quiet }) : null
+    this.renderOpts = { dir: this.dir, dev, staticMarkup, hotReloader: this.hotReloader }
     this.http = null
     this.config = getConfig(this.dir)
 
@@ -107,6 +107,7 @@ export default class Server {
 
         const paths = params.path || ['index']
         const pathname = `/${paths.join('/')}`
+
         await this.renderJSON(req, res, pathname)
       },
 

server/on-demand-entry-handler.js

@@ -0,0 +1,184 @@
+import DynamicEntryPlugin from 'webpack/lib/DynamicEntryPlugin'
+import { EventEmitter } from 'events'
+import { join } from 'path'
+import { parse } from 'url'
+import resolvePath from './resolve'
+import touch from 'touch'
+
+const ADDED = Symbol()
+const BUILDING = Symbol()
+const BUILT = Symbol()
+
+export default function onDemandEntryHandler (devMiddleware, compiler, {
+  dir,
+  dev,
+  maxInactiveAge = 1000 * 25
+}) {
+  const entries = {}
+  const lastAccessPages = ['']
+  const doneCallbacks = new EventEmitter()
+  let touchedAPage = false
+
+  compiler.plugin('make', function (compilation, done) {
+    const allEntries = Object.keys(entries).map((page) => {
+      const { name, entry } = entries[page]
+      entries[page].status = BUILDING
+      return addEntry(compilation, this.context, name, entry)
+    })
+
+    Promise.all(allEntries)
+      .then(() => done())
+      .catch(done)
+  })
+
+  compiler.plugin('done', function (stats) {
+    // Call all the doneCallbacks
+    Object.keys(entries).forEach((page) => {
+      const entryInfo = entries[page]
+      if (entryInfo.status !== BUILDING) return
+
+      // With this, we are triggering a filesystem based watch trigger
+      // It'll memorize some timestamp related info related to common files used
+      // in the page
+      // That'll reduce the page building time significantly.
+      if (!touchedAPage) {
+        setTimeout(() => {
+          touch.sync(entryInfo.pathname)
+        }, 0)
+        touchedAPage = true
+      }
+
+      entryInfo.status = BUILT
+      entries[page].lastActiveTime = Date.now()
+      doneCallbacks.emit(page)
+    })
+  })
+
+  setInterval(function () {
+    disposeInactiveEntries(devMiddleware, entries, lastAccessPages, maxInactiveAge)
+  }, 5000)
+
+  return {
+    async ensurePage (page) {
+      page = normalizePage(page)
+
+      const pagePath = join(dir, 'pages', page)
+      const pathname = await resolvePath(pagePath)
+      const name = join('bundles', pathname.substring(dir.length))
+
+      const entry = [
+        join(__dirname, '..', 'client/webpack-hot-middleware-client'),
+        join(__dirname, '..', 'client', 'on-demand-entries-client'),
+        `${pathname}?entry`
+      ]
+
+      await new Promise((resolve, reject) => {
+        const entryInfo = entries[page]
+
+        if (entryInfo) {
+          if (entryInfo.status === BUILT) {
+            resolve()
+            return
+          }
+
+          if (entryInfo.status === BUILDING) {
+            doneCallbacks.on(page, processCallback)
+            return
+          }
+        }
+
+        console.log(`> Building page: ${page}`)
+
+        entries[page] = { name, entry, pathname, status: ADDED }
+        doneCallbacks.on(page, processCallback)
+
+        devMiddleware.invalidate()
+
+        function processCallback (err) {
+          if (err) return reject(err)
+          resolve()
+        }
+      })
+    },
+
+    middleware () {
+      return function (req, res, next) {
+        if (!/^\/on-demand-entries-ping/.test(req.url)) return next()
+
+        const { query } = parse(req.url, true)
+        const page = normalizePage(query.page)
+        const entryInfo = entries[page]
+
+        // If there's no entry.
+        // Then it seems like an weird issue.
+        if (!entryInfo) {
+          const message = `Client pings, but there's no entry for page: ${page}`
+          console.error(message)
+          sendJson(res, { invalid: true })
+          return
+        }
+
+        // We don't need to maintain active state of anything other than BUILT entries
+        if (entryInfo.status !== BUILT) return
+
+        // If there's an entryInfo
+        lastAccessPages.pop()
+        lastAccessPages.unshift(page)
+        entryInfo.lastActiveTime = Date.now()
+
+        sendJson(res, { success: true })
+      }
+    }
+  }
+}
+
+function addEntry (compilation, context, name, entry) {
+  return new Promise((resolve, reject) => {
+    const dep = DynamicEntryPlugin.createDependency(entry, name)
+    compilation.addEntry(context, dep, name, (err) => {
+      if (err) return reject(err)
+      resolve()
+    })
+  })
+}
+
+function disposeInactiveEntries (devMiddleware, entries, lastAccessPages, maxInactiveAge) {
+  const disposingPages = []
+
+  Object.keys(entries).forEach((page) => {
+    const { lastActiveTime, status } = entries[page]
+
+    // This means this entry is currently building or just added
+    // We don't need to dispose those entries.
+    if (status !== BUILT) return
+
+    // We should not build the last accessed page even we didn't get any pings
+    // Sometimes, it's possible our XHR ping to wait before completing other requests.
+    // In that case, we should not dispose the current viewing page
+    if (lastAccessPages[0] === page) return
+
+    if (Date.now() - lastActiveTime > maxInactiveAge) {
+      disposingPages.push(page)
+    }
+  })
+
+  if (disposingPages.length > 0) {
+    disposingPages.forEach((page) => {
+      delete entries[page]
+    })
+    console.log(`> Disposing inactive page(s): ${disposingPages.join(', ')}`)
+    devMiddleware.invalidate()
+  }
+}
+
+// /index and / is the same. So, we need to identify both pages as the same.
+// This also applies to sub pages as well.
+function normalizePage (page) {
+  return page.replace(/\/index$/, '/')
+}
+
+function sendJson (res, payload) {
+  res.setHeader('Content-Type', 'application/json')
+  res.status = 200
+  res.end(JSON.stringify(payload))
+}