Issue #11257(Updated) - Change build process to include npm pack and …

…unpacking (#11750)

* Change build process to include npm pack and unpacking generated packages to corresponding build directories.

* Update function name, change to use os's default temp directory

* appending uuid to temp npm packaging directory.

package.json

@@ -89,9 +89,11 @@
     "rollup-plugin-replace": "^1.1.1",
     "rollup-plugin-strip-banner": "^0.2.0",
     "run-sequence": "^1.1.4",
+    "targz": "^1.0.1",
     "through2": "^2.0.0",
     "tmp": "~0.0.28",
     "typescript": "~1.8.10",
+    "uuid": "^3.1.0",
     "yargs": "^6.3.0"
   },
   "devEngines": {

scripts/rollup/build.js

@@ -10,6 +10,7 @@ const stripBanner = require('rollup-plugin-strip-banner');
 const chalk = require('chalk');
 const path = require('path');
 const resolve = require('rollup-plugin-node-resolve');
+const os = require('os');
 const fs = require('fs');
 const rimraf = require('rimraf');
 const argv = require('minimist')(process.argv.slice(2));
@@ -26,6 +27,7 @@ const syncReactNativeCS = require('./sync').syncReactNativeCS;
 const Packaging = require('./packaging');
 const codeFrame = require('babel-code-frame');
 const Wrappers = require('./wrappers');
+const uuidv1 = require('uuid/v1');
 
 const UMD_DEV = Bundles.bundleTypes.UMD_DEV;
 const UMD_PROD = Bundles.bundleTypes.UMD_PROD;
@@ -48,6 +50,10 @@ const shouldExtractErrors = argv['extract-errors'];
 const errorCodeOpts = {
   errorMapFilePath: 'scripts/error-codes/codes.json',
 };
+const npmPackagesTmpDir = path.join(
+  os.tmpdir(),
+  `react-npm-packages-${uuidv1()}`
+);
 
 const closureOptions = {
   compilationLevel: 'SIMPLE',
@@ -397,7 +403,12 @@ async function createBundle(bundle, bundleType) {
         bundle.moduleType
       )
     );
-    await Packaging.createNodePackage(bundleType, packageName, filename);
+    await Packaging.createNodePackage(
+      bundleType,
+      packageName,
+      filename,
+      npmPackagesTmpDir
+    );
     console.log(`${chalk.bgGreen.black(' COMPLETE ')} ${logKey}\n`);
   } catch (error) {
     if (error.code) {
@@ -435,6 +446,9 @@ rimraf('build', async () => {
   try {
     // create a new build directory
     fs.mkdirSync('build');
+    // create the temp directory for local npm packing and unpacking
+    // in operating system's default temporary directory
+    fs.mkdirSync(npmPackagesTmpDir);
     // create the packages folder for NODE+UMD bundles
     fs.mkdirSync(path.join('build', 'packages'));
     // create the dist folder for UMD bundles
@@ -476,6 +490,12 @@ rimraf('build', async () => {
           'used when the error map needs to be rebuilt.\n'
       );
     }
+    rimraf(npmPackagesTmpDir, err => {
+      if (err) {
+        console.error(err);
+        process.exit(1);
+      }
+    });
   } catch (err) {
     console.error(err);
     process.exit(1);

scripts/rollup/packaging.js

@@ -6,6 +6,8 @@ const join = require('path').join;
 const resolve = require('path').resolve;
 const Bundles = require('./bundles');
 const asyncCopyTo = require('./utils').asyncCopyTo;
+const asyncExecuteCommand = require('./utils').asyncExecuteCommand;
+const asyncExtractTar = require('./utils').asyncExtractTar;
 
 const UMD_DEV = Bundles.bundleTypes.UMD_DEV;
 const UMD_PROD = Bundles.bundleTypes.UMD_PROD;
@@ -84,8 +86,13 @@ async function createFacebookWWWBuild() {
   await asyncCopyTo(from, to);
 }
 
-async function copyBundleIntoNodePackage(packageName, filename, bundleType) {
-  const packageDirectory = resolve(`./build/packages/${packageName}`);
+async function copyBundleIntoNodePackage(
+  packageName,
+  filename,
+  bundleType,
+  npmPackagesTmpDir
+) {
+  const packageDirectory = resolve(`${npmPackagesTmpDir}/${packageName}`);
   if (!fs.existsSync(packageDirectory)) {
     return;
   }
@@ -121,9 +128,9 @@ async function copyBundleIntoNodePackage(packageName, filename, bundleType) {
   }
 }
 
-async function copyNodePackageTemplate(packageName) {
+async function copyNodePackageTemplate(packageName, npmPackagesTmpDir) {
   const from = resolve(`./packages/${packageName}`);
-  const to = resolve(`./build/packages/${packageName}`);
+  const to = resolve(`${npmPackagesTmpDir}/${packageName}`);
   const npmFrom = resolve(`${from}/npm`);
   if (!fs.existsSync(npmFrom)) {
     // The package is not meant for npm consumption.
@@ -133,21 +140,50 @@ async function copyNodePackageTemplate(packageName) {
     // We already created this package (e.g. due to another entry point).
     return;
   }
-  // TODO: verify that all copied files are either in the "files"
-  // whitelist or implicitly published by npm.
   await asyncCopyTo(npmFrom, to);
   await asyncCopyTo(resolve(`${from}/package.json`), `${to}/package.json`);
   await asyncCopyTo(resolve(`${from}/README.md`), `${to}/README.md`);
   await asyncCopyTo(resolve('./LICENSE'), `${to}/LICENSE`);
 }
 
-async function createNodePackage(bundleType, packageName, filename) {
+async function packForNpmAndUnpack(packageName, npmPackagesTmpDir) {
+  const packageTmpDir = resolve(`${npmPackagesTmpDir}/${packageName}`);
+  const extractTmpDir = resolve(`${packageTmpDir}/extract`);
+  const build = resolve(`./build/packages/${packageName}`);
+  const npmFrom = resolve(`./packages/${packageName}/npm`);
+  if (!fs.existsSync(npmFrom)) {
+    return;
+  }
+  let tgzName = await asyncExecuteCommand(`cd ${packageTmpDir} && npm pack`);
+  // In npm packages, files are grouped into a root directory(named 'package').
+  // We only copy the packed files instead of extract the root 'package' directly to build directory
+  await asyncExtractTar({
+    src: `${packageTmpDir}/${tgzName.trim()}`,
+    dest: extractTmpDir,
+  });
+  await asyncCopyTo(`${extractTmpDir}/package`, build);
+}
+
+async function createNodePackage(
+  bundleType,
+  packageName,
+  filename,
+  npmPackagesTmpDir
+) {
   // the only case where we don't want to copy the package is for FB bundles
   if (bundleType === FB_DEV || bundleType === FB_PROD) {
     return;
   }
-  await copyNodePackageTemplate(packageName);
-  await copyBundleIntoNodePackage(packageName, filename, bundleType);
+  await copyNodePackageTemplate(packageName, npmPackagesTmpDir);
+  await copyBundleIntoNodePackage(
+    packageName,
+    filename,
+    bundleType,
+    npmPackagesTmpDir
+  );
+  // Packing packages locally, simulate npm publish,
+  // Then unpacking generated packages to build directory
+  await packForNpmAndUnpack(packageName, npmPackagesTmpDir);
 }
 
 function getOutputPathRelativeToBuildFolder(bundleType, filename, hasteName) {

scripts/rollup/utils.js

@@ -3,6 +3,8 @@
 const ncp = require('ncp').ncp;
 const join = require('path').join;
 const resolve = require('path').resolve;
+const exec = require('child_process').exec;
+const targz = require('targz');
 
 function asyncCopyTo(from, to) {
   return new Promise(_resolve => {
@@ -24,7 +26,35 @@ function resolvePath(path) {
   }
 }
 
+function asyncExecuteCommand(command) {
+  return new Promise(_resolve =>
+    exec(command, (error, stdout) => {
+      if (!error) {
+        _resolve(stdout);
+      } else {
+        console.error(error);
+        process.exit(1);
+      }
+    })
+  );
+}
+
+function asyncExtractTar(options) {
+  return new Promise(_resolve =>
+    targz.decompress(options, error => {
+      if (!error) {
+        _resolve();
+      } else {
+        console.error(error);
+        process.exit(1);
+      }
+    })
+  );
+}
+
 module.exports = {
   asyncCopyTo: asyncCopyTo,
   resolvePath: resolvePath,
+  asyncExecuteCommand: asyncExecuteCommand,
+  asyncExtractTar: asyncExtractTar,
 };

yarn.lock

@@ -935,6 +935,12 @@ binary-extensions@^1.0.0:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.8.0.tgz#48ec8d16df4377eae5fa5884682480af4d95c774"
 
+bl@^1.0.0:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.1.tgz#cac328f7bee45730d404b692203fcb590e172d5e"
+  dependencies:
+    readable-stream "^2.0.5"
+
 block-stream@*:
   version "0.0.9"
   resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a"
@@ -1101,6 +1107,10 @@ chokidar@^1.0.0, chokidar@^1.6.1:
   optionalDependencies:
     fsevents "^1.0.0"
 
+chownr@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.0.1.tgz#e2a75042a9551908bebd25b8523d5f9769d79181"
+
 ci-info@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-1.0.0.tgz#dc5285f2b4e251821683681c381c3388f46ec534"
@@ -1529,7 +1539,7 @@ end-of-stream@1.0.0:
   dependencies:
     once "~1.3.0"
 
-end-of-stream@^1.1.0:
+end-of-stream@^1.0.0, end-of-stream@^1.1.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.0.tgz#7a90d833efda6cfa6eac0f4949dbb0fad3a63206"
   dependencies:
@@ -2387,7 +2397,7 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@2, inherits@^2.0.1, inherits@~2.0.0, inherits@~2.0.1:
+inherits@2, inherits@^2.0.1, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
 
@@ -3868,6 +3878,18 @@ readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable
     string_decoder "~1.0.0"
     util-deprecate "~1.0.1"
 
+readable-stream@^2.0.5:
+  version "2.3.3"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c"
+  dependencies:
+    core-util-is "~1.0.0"
+    inherits "~2.0.3"
+    isarray "~1.0.0"
+    process-nextick-args "~1.0.6"
+    safe-buffer "~5.1.1"
+    string_decoder "~1.0.3"
+    util-deprecate "~1.0.1"
+
 readable-stream@~2.0.0:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e"
@@ -4227,6 +4249,10 @@ safe-buffer@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7"
 
+safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
+
 sane@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/sane/-/sane-2.0.0.tgz#99cb79f21f4a53a69d4d0cd957c2db04024b8eb2"
@@ -4445,6 +4471,12 @@ string_decoder@~1.0.0:
   dependencies:
     buffer-shims "~1.0.0"
 
+string_decoder@~1.0.3:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab"
+  dependencies:
+    safe-buffer "~5.1.0"
+
 stringmap@~0.2.2:
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/stringmap/-/stringmap-0.2.2.tgz#556c137b258f942b8776f5b2ef582aa069d7d1b1"
@@ -4526,6 +4558,15 @@ table@^3.7.8:
     slice-ansi "0.0.4"
     string-width "^2.0.0"
 
+tar-fs@^1.8.1:
+  version "1.16.0"
+  resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-1.16.0.tgz#e877a25acbcc51d8c790da1c57c9cf439817b896"
+  dependencies:
+    chownr "^1.0.1"
+    mkdirp "^0.5.1"
+    pump "^1.0.0"
+    tar-stream "^1.1.2"
+
 tar-pack@^3.4.0:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.0.tgz#23be2d7f671a8339376cbdb0b8fe3fdebf317984"
@@ -4539,6 +4580,15 @@ tar-pack@^3.4.0:
     tar "^2.2.1"
     uid-number "^0.0.6"
 
+tar-stream@^1.1.2:
+  version "1.5.5"
+  resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.5.5.tgz#5cad84779f45c83b1f2508d96b09d88c7218af55"
+  dependencies:
+    bl "^1.0.0"
+    end-of-stream "^1.0.0"
+    readable-stream "^2.0.0"
+    xtend "^4.0.0"
+
 tar@^2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1"
@@ -4547,6 +4597,12 @@ tar@^2.2.1:
     fstream "^1.0.2"
     inherits "2"
 
+targz@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/targz/-/targz-1.0.1.tgz#8f76a523694cdedfbb5d60a4076ff6eeecc5398f"
+  dependencies:
+    tar-fs "^1.8.1"
+
 test-exclude@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-4.1.1.tgz#4d84964b0966b0087ecc334a2ce002d3d9341e26"
@@ -4732,6 +4788,10 @@ uuid@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
 
+uuid@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"
+
 v8flags@^2.0.10:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/v8flags/-/v8flags-2.1.1.tgz#aab1a1fa30d45f88dd321148875ac02c0b55e5b4"