feat(gatsby-dev-cli): add verdaccio support

packages/gatsby-dev-cli/.gitignore

@@ -28,3 +28,9 @@ node_modules
 
 decls
 dist
+
+# verdaccio local storage
+verdaccio/storage
+# autogenerated by verdaccio (?)
+/.htpasswd
+

packages/gatsby-dev-cli/package.json

@@ -16,6 +16,8 @@
     "fs-extra": "^4.0.1",
     "is-absolute": "^0.2.6",
     "lodash": "^4.17.10",
+    "npm-cli-login": "^0.1.1",
+    "verdaccio": "^3.11.1",
     "yargs": "^8.0.2"
   },
   "devDependencies": {
@@ -36,6 +38,5 @@
     "prepare": "cross-env NODE_ENV=production npm run build",
     "test": "echo \"Error: no test specified\" && exit 1",
     "watch": "babel -w src --out-dir dist"
-  },
-  "gitHead": "5bd5aebe066b9875354a81a4b9ed98722731c465"
+  }
 }

packages/gatsby-dev-cli/src/utils.js

@@ -0,0 +1,20 @@
+const child_process = require(`child_process`)
+
+exports.promisifiedSpawn = ([cmd, args = [], spawnArgs = {}]) =>
+  new Promise((resolve, reject) => {
+    const proc = child_process.spawn(cmd, args, spawnArgs)
+
+    proc.stdout.setEncoding(`utf8`)
+    proc.stdout.on(`data`, console.log)
+
+    proc.stderr.setEncoding(`utf8`)
+    proc.stderr.on(`data`, console.error)
+
+    proc.on(`close`, code => {
+      if (code) {
+        reject(code)
+      } else {
+        resolve()
+      }
+    })
+  })

packages/gatsby-dev-cli/src/verdaccio.js

@@ -0,0 +1,137 @@
+const startVerdaccio = require(`verdaccio`).default
+const path = require(`path`)
+const npmLogin = require(`npm-cli-login`)
+const fs = require(`fs-extra`)
+const { promisifiedSpawn } = require(`./utils`)
+
+let VerdaccioInitPromise = null
+
+const verdaccioConfig = {
+  storage: path.join(__dirname, `..`, `verdaccio`, `storage`),
+  port: 4873, // default
+  web: {
+    enable: true,
+    title: `gatsby-dev`,
+  },
+  auth: {
+    htpasswd: {
+      file: `./.htpasswd`,
+      max_users: 1000,
+    },
+  },
+  packages: {
+    "**": {
+      access: `$all`,
+      publish: `gatsby-dev`, // we create dummy gatsby-dev username
+      proxy: `npmjs`,
+    },
+  },
+  uplinks: {
+    npmjs: {
+      url: `https://registry.npmjs.org/`,
+    },
+  },
+  self_path: path.join(__dirname, `..`, `verdaccio`),
+}
+
+const registryUrl = `http://localhost:${verdaccioConfig.port}`
+
+exports.registryUrl = registryUrl
+
+const startServer = () => {
+  if (VerdaccioInitPromise) {
+    return VerdaccioInitPromise
+  }
+
+  console.log(`Starting local verdaccio server`)
+
+  // clear storage
+  fs.removeSync(verdaccioConfig.storage)
+
+  VerdaccioInitPromise = new Promise(resolve => {
+    startVerdaccio(
+      verdaccioConfig,
+      verdaccioConfig.port,
+      verdaccioConfig.storage,
+      `1.0.0`,
+      `gatsby-dev`,
+      (webServer, addr, pkgName, pkgVersion) => {
+        webServer.listen(addr.port || addr.path, addr.host, () => {
+          console.log(`Started local verdaccio server`)
+
+          // Login with dummy gatsby-dev user - see  ./verdaccio/.htpasswd .
+          // This is needed to be able to actually publish packages
+          // to local registry.
+          console.log(`Logging in with dummy user`)
+
+          npmLogin(
+            `gatsby-dev`,
+            `gatsby-dev`,
+            `gatsby-dev@gatsbyjs.org`,
+            registryUrl
+          )
+
+          // This is silly - `npm-cli-login` doesn't return promise or callback
+          // so let's just wait reasaonable amount before resolving
+          // There is open pull request to add returning promise:
+          // https://github.com/postmanlabs/npm-cli-login/pull/19
+          setTimeout(() => {
+            console.log(`Hopefully logged in already`)
+            resolve()
+          }, 1500)
+        })
+      }
+    )
+  })
+
+  return VerdaccioInitPromise
+}
+
+exports.publishPackageLocally = async ({
+  pathToPackage,
+  packageName,
+  version,
+}) => {
+  // <ake sure verdaccio is running
+  await startServer()
+
+  // npm unpublish
+  // because version doesn't change when developing locally verdaccio wouldn't
+  // let republish package with same version - so as quick workaround we
+  // unpublish package.
+  // Ideally we could check if we need to unpublish: i.e. compare deps of
+  // published package to deps of package in monorepo.
+  // Also for CI we could make special case to skip this, because then
+  // there is no way to edit source files and we can use fast path.
+  const unpublishCmd = [
+    `npm`,
+    [`unpublish`, `${packageName}@${version}`, `--registry=${registryUrl}`],
+  ]
+
+  console.log(
+    `Trying to unpublish ${packageName}@${version} to local registry, in case it was published before`
+  )
+  try {
+    await promisifiedSpawn(unpublishCmd)
+    console.log(`Unpublished ${packageName}@${version} from local registry`)
+  } catch {
+    console.log(
+      `Didn't unpublish ${packageName}@${version} - probably package wasn't published before`
+    )
+  }
+
+  // npm publish
+  const publishCmd = [
+    `npm`,
+    [`publish`, `--registry=${registryUrl}`],
+    {
+      cwd: pathToPackage,
+    },
+  ]
+
+  console.log(`Publishing ${packageName}@${version} to local registry`)
+
+  await promisifiedSpawn(publishCmd)
+
+  console.log(`Published ${packageName}@${version} to local registry`)
+}

packages/gatsby-dev-cli/src/watch.js

@@ -3,6 +3,9 @@ const _ = require(`lodash`)
 const fs = require(`fs-extra`)
 const path = require(`path`)
 
+const { publishPackageLocally, registryUrl } = require(`./verdaccio`)
+const { promisifiedSpawn } = require(`./utils`)
+
 let numCopied = 0
 
 const quit = () => {
@@ -26,6 +29,21 @@ const copyPath = (oldPath, newPath, quiet) =>
     })
   })
 
+const installFromVerdaccio = async args => {
+  await publishPackageLocally(args)
+
+  const installCmd = [
+    `yarn`,
+    [`add`, args.packageName, `--registry=${registryUrl}`],
+  ]
+
+  console.log(`Installing '${args.packageName}' locally`)
+
+  await promisifiedSpawn(installCmd)
+
+  console.log(`Installed '${args.packageName}' locally`)
+}
+
 /*
  * non-existant packages break on('ready')
  * See: https://github.com/paulmillr/chokidar/issues/449
@@ -61,11 +79,67 @@ function watch(root, packages, { scanOnce, quiet }) {
           return
         }
 
+        const relativePackageFile = path.relative(prefix, filePath)
+
         const newPath = path.join(
           `./node_modules/${packageName}`,
-          path.relative(prefix, filePath)
+          relativePackageFile
         )
 
+        if (relativePackageFile === `package.json`) {
+          // compare dependencies with local version
+
+          try {
+            // Delete require cache so we can reload the module.
+            delete require.cache[require.resolve(filePath)]
+            // eslint-disable-next-line
+          } catch {}
+          try {
+            // Delete require cache so we can reload the module.
+            delete require.cache[
+              require.resolve(path.join(process.cwd(), newPath))
+            ]
+            // eslint-disable-next-line
+          } catch {}
+
+          const monorepoPKGjson = require(filePath)
+
+          let localPKGjson
+          try {
+            localPKGjson = require(path.join(process.cwd(), newPath))
+          } catch {
+            localPKGjson = {
+              dependencies: {},
+            }
+          }
+
+          const areDepsEqual = _.isEqual(
+            monorepoPKGjson.dependencies,
+            localPKGjson.dependencies
+          )
+
+          console.log({
+            packageName,
+            filePath,
+            localPackage: path.join(process.cwd(), newPath),
+            areDepsEqual,
+            mDeps: monorepoPKGjson.dependencies,
+            lDeps: localPKGjson.dependencies,
+          })
+
+          if (!areDepsEqual) {
+            allCopies.push(
+              installFromVerdaccio({
+                pathToPackage: prefix,
+                packageName,
+                version: monorepoPKGjson.version,
+              })
+            )
+          }
+
+          return
+        }
+
         let localCopies = [copyPath(filePath, newPath, quiet)]
 
         // If this is from "cache-dir" also copy it into the site's .cache

packages/gatsby-dev-cli/verdaccio/.htpasswd

@@ -0,0 +1 @@
+gatsby-dev:$apr1$vh8SxK9U$4yf.LQ1.u5TftWGfAXQ7q.

packages/gatsby/package.json

@@ -71,7 +71,7 @@
     "graphql-playground-middleware-express": "^1.7.10",
     "graphql-relay": "^0.6.0",
     "graphql-skip-limit": "^2.0.5",
-    "graphql-tools": "^3.0.4",
+    "graphql-tools": "^4.0.3",
     "graphql-type-json": "^0.2.1",
     "hash-mod": "^0.0.5",
     "invariant": "^2.2.4",
@@ -165,7 +165,8 @@
     "url": "git+https://github.com/gatsbyjs/gatsby.git"
   },
   "resolutions": {
-    "graphql": "^14.1.1"
+    "graphql": "^14.1.1",
+    "terser": "3.14.1"
   },
   "scripts": {
     "build": "npm run build:src && npm run build:internal-plugins && npm run build:rawfiles && npm run build:cjs",

scripts/e2e-test.sh

@@ -3,13 +3,15 @@ SRC_PATH=$1
 CUSTOM_COMMAND="${2:-yarn test}"
 GATSBY_PATH="${CIRCLE_WORKING_DIRECTORY:-../../}"
 
-npm install -g gatsby-dev-cli &&
+# npm install -g gatsby-dev-cli &&
 
 # setting up child integration test link to gatsby packages
 cd $SRC_PATH &&
 yarn &&
-gatsby-dev --set-path-to-repo $GATSBY_PATH &&
-gatsby-dev --scan-once --copy-all --quiet && # copies _all_ files in gatsby/packages
+#gatsby-dev --set-path-to-repo $GATSBY_PATH &&
+#gatsby-dev --scan-once --copy-all --quiet && # copies _all_ files in gatsby/packages
+node ../../packages/gatsby-dev-cli/dist/index.js --set-path-to-repo $GATSBY_PATH &&
+node ../../packages/gatsby-dev-cli/dist/index.js --scan-once --quiet && # copies _all_ files in gatsby/packages
 chmod +x ./node_modules/.bin/gatsby && # this is sometimes necessary to ensure executable
 sh -c "$CUSTOM_COMMAND" &&
 echo "e2e test run succeeded"