Set up CI with Azure Pipelines (#6495)

* Set up CI with Azure Pipelines

* Update azure-pipelines.yml

* Update azure-pipelines.yml

* Update package.json

* Adds debug

* Tries using bash

* Tweaks timeouts

* Fixes various things related to the win32 compat

* I get the same 22 failed tests with/without the delay.

* Fix makeFakeBinary for win32.

It needs the ".cmd" extension, otherwise it's ignored.
Remove #!/bin/sh and instead add @echo off to clean output
Use errorCode 0 so we get output and not a Command failed

* Check custom output instead of versions.

Running fake binaries there was no way to get local versions of node or yarn. Am I missing something?

* Remove quotes from echo (not removed in Windows).
Replace ; by /n as the former doesn't actually break statements in Windows batch files.

* Some tweaks

* Fixes syntax

* Fixes tests on Windows

* Adds test reporting

* Fixes things (?)

* Fixes indent

* Always run the test result publishing

* Fixes yml

* Use node scripts to avoid Windows interoperability issues.

* Fixes the "must use the same Node as Yarn itself" test

* Fixes escaping from within the tests

* Removes the accidental .only flag

* Fixes a test on Windows

* Add support for different drives in Windows.

So if code is in D: but temporary folder is in C:, don't try to find a relative path but stay on the absolute one.

* No need to getPackageLocation

* Add support for environment and arguments by using a custom solution instead of cmd-shim.

* Missing escape on path in pnpapi test.

* Some cleaning

* Runs tests on all three platforms

* Fixes write paths

* Fixes normalizePath

* Disables pkg-tests from CircleCI (moved to Azure)

* Increase Windows timeouts and better name Test results (#6680)

* Increase timeout in Windows, we're seeing tests failing randomly and others close to default 5 sec.

* Distinguish tests published from each job.

* Pass name as vmImage is not available

* Remove unnecessary detect unfinished tests.

* Using strategy var instead of parameter

* Use variables instead of strategy

* Revert "Disables pkg-tests from CircleCI (moved to Azure)"

This reverts commit 8f72462.

azure-pipelines.yml

@@ -0,0 +1,51 @@
+jobs:
+
+- job: Windows
+  pool: 'Hosted VS2017'
+
+  variables:
+    os_name: Windows
+
+  strategy:
+    matrix:
+      node_8_x:
+        node_version: 8.x
+      node_10_x:
+        node_version: 10.x
+
+  steps:
+  - template: scripts/azure-run-tests.yml
+
+- job: Linux
+  pool:
+    vmImage: 'Ubuntu 16.04'
+
+  variables:
+    os_name: Linux
+
+  strategy:
+    matrix:
+      node_8_x:
+        node_version: 8.x
+      node_10_x:
+        node_version: 10.x
+
+  steps:
+  - template: scripts/azure-run-tests.yml
+
+- job: OSX
+  pool:
+    vmImage: 'macOS 10.13'
+
+  variables:
+    os_name: OSX
+
+  strategy:
+    matrix:
+      node_8_x:
+        node_version: 8.x
+      node_10_x:
+        node_version: 10.x
+
+  steps:
+  - template: scripts/azure-run-tests.yml

packages/pkg-tests/.gitignore

@@ -1,2 +1,3 @@
 node_modules
 yarn-error.log
+junit.xml

packages/pkg-tests/package.json

@@ -11,6 +11,7 @@
         "babel-preset-flow": "^6.23.0",
         "flow-bin": "^0.66.0",
         "jest": "^23.0.0",
+        "jest-junit": "^5.2.0",
         "prettier": "^1.10.2"
     },
     "scripts": {
@@ -25,7 +26,7 @@
     },
     "jest": {
         "modulePathIgnorePatterns": [
-            "<rootDir>[/\\\\]pkg-tests-fixtures[/\\\\]packages[/\\\\]"
+            "<rootDir>/pkg-tests-fixtures/packages/"
         ]
     }
 }

packages/pkg-tests/pkg-tests-core/sources/utils/exec.js

@@ -12,6 +12,8 @@ exports.execFile = function(
       if (error) {
         reject(error);
       } else {
+        stdout = stdout.replace(/\r\n?/g, `\n`);
+        stderr = stderr.replace(/\r\n?/g, `\n`);
         resolve({stdout, stderr});
       }
     });

packages/pkg-tests/pkg-tests-core/sources/utils/fs.js

@@ -11,6 +11,8 @@ const zlib = require('zlib');
 
 const miscUtils = require('./misc');
 
+const IS_WIN32 = process.platform === `win32`;
+
 exports.walk = function walk(
   source: string,
   {filter, relative = false}: {|filter?: Array<string>, relative?: boolean|} = {},
@@ -75,13 +77,13 @@ exports.packToStream = function packToStream(
   const packStream = tarFs.pack(source, {
     map: header => {
       if (true) {
-        header.name = path.resolve('/', header.name);
-        header.name = path.relative('/', header.name);
+        header.name = path.posix.resolve('/', header.name);
+        header.name = path.posix.relative('/', header.name);
       }
 
       if (virtualPath) {
-        header.name = path.resolve('/', virtualPath, header.name);
-        header.name = path.relative('/', header.name);
+        header.name = path.posix.resolve('/', virtualPath, header.name);
+        header.name = path.posix.relative('/', header.name);
       }
 
       return header;
@@ -186,6 +188,9 @@ exports.makeFakeBinary = async function(
   target: string,
   {output = `Fake binary`, exitCode = 1}: {|output: string, exitCode: number|} = {},
 ): Promise<void> {
-  await exports.writeFile(target, `#!/bin/sh\necho "${output}"\nexit ${exitCode}\n`);
-  await exports.chmod(target, 0o755);
+  const realTarget = IS_WIN32 ? `${target}.cmd` : target;
+  const header = IS_WIN32 ? `@echo off\n` : `#!/bin/sh\n`;
+
+  await exports.writeFile(realTarget, `${header}echo ${output}\nexit ${exitCode}\n`);
+  await exports.chmod(realTarget, 0o755);
 };

packages/pkg-tests/pkg-tests-core/sources/utils/tests.js

@@ -364,3 +364,9 @@ exports.generatePkgDriver = function generatePkgDriver({runDriver}: {|runDriver:
 
   return withConfig({});
 };
+
+exports.testIf = function testIf(condition, ...args) {
+  if (condition()) {
+    test(...args);
+  }
+};

packages/pkg-tests/pkg-tests-fixtures/packages/no-deps-scripted-1.0.0/package.json

@@ -2,8 +2,8 @@
     "name": "no-deps-scripted",
     "version": "1.0.0",
     "scripts": {
-        "preinstall": "echo 'module.exports.push(100);' >> log.js",
-        "install": "echo 'module.exports.push(200);' >> log.js",
-        "postinstall": "echo 'module.exports.push(300);' >> log.js; echo 'module.exports = '\"$(node -p 'Math.floor(Math.random() * 512000)')\"';' > rnd.js"
+        "preinstall": "node scripts/preinstall",
+        "install": "node scripts/install",
+        "postinstall": "node scripts/postinstall"
     }
 }

packages/pkg-tests/pkg-tests-fixtures/packages/no-deps-scripted-1.0.0/scripts/install.js

@@ -0,0 +1,3 @@
+const fs = require(`fs`);
+
+fs.appendFileSync(`${__dirname}/../log.js`, `module.exports.push('install');`);

packages/pkg-tests/pkg-tests-fixtures/packages/no-deps-scripted-1.0.0/scripts/postinstall.js

@@ -0,0 +1,4 @@
+const fs = require(`fs`);
+
+fs.appendFileSync(`${__dirname}/../log.js`, `module.exports.push('postinstall');`);
+fs.appendFileSync(`${__dirname}/../rnd.js`, `module.exports = ${Math.floor(Math.random() * 512000)};`);

packages/pkg-tests/pkg-tests-fixtures/packages/no-deps-scripted-1.0.0/scripts/preinstall.js

@@ -0,0 +1,3 @@
+const fs = require(`fs`);
+
+fs.appendFileSync(`${__dirname}/../log.js`, `module.exports.push('preinstall');`);

packages/pkg-tests/pkg-tests-specs/sources/pnp.js

@@ -5,7 +5,7 @@ const {satisfies} = require('semver');
 
 const {
   fs: {createTemporaryFolder, readFile, readJson, writeFile, writeJson},
-  tests: {getPackageDirectoryPath},
+  tests: {getPackageDirectoryPath, testIf},
 } = require('pkg-tests-core');
 
 module.exports = makeTemporaryEnv => {
@@ -312,7 +312,7 @@ module.exports = makeTemporaryEnv => {
       makeTemporaryEnv(
         {
           dependencies: {[`no-deps`]: `1.0.0`},
-          scripts: {myScript: `node -p 'require("no-deps/package.json").version'`},
+          scripts: {myScript: `node -p "require('no-deps/package.json').version"`},
         },
         {
           plugNPlay: true,
@@ -574,7 +574,7 @@ module.exports = makeTemporaryEnv => {
 
         await writeFile(`${tmp}/folder/index.js`, `module.exports = 42;`);
 
-        await expect(source(`require("${tmp}/folder")`)).resolves.toEqual(42);
+        await expect(source(`require(${JSON.stringify(tmp)} + "/folder")`)).resolves.toEqual(42);
       }),
     );
 
@@ -587,7 +587,7 @@ module.exports = makeTemporaryEnv => {
 
         await writeFile(`${tmp}/file.js`, `module.exports = 42;`);
 
-        await expect(source(`require("${tmp}/file")`)).resolves.toEqual(42);
+        await expect(source(`require(${JSON.stringify(tmp)} + "/file")`)).resolves.toEqual(42);
       }),
     );
 
@@ -621,7 +621,7 @@ module.exports = makeTemporaryEnv => {
         await writeFile(`${tmp}/node_modules/dep/index.js`, `module.exports = 42;`);
         await writeFile(`${tmp}/index.js`, `require('dep')`);
 
-        await source(`require("${tmp}/index.js")`);
+        await source(`require(${JSON.stringify(tmp)} + "/index.js")`);
       }),
     );
 
@@ -744,7 +744,8 @@ module.exports = makeTemporaryEnv => {
       ),
     );
 
-    test(
+    testIf(
+      () => process.platform !== 'win32',
       `it should generate a file that can be used as an executable to resolve a request (valid request)`,
       makeTemporaryEnv(
         {
@@ -773,7 +774,8 @@ module.exports = makeTemporaryEnv => {
       ),
     );
 
-    test(
+    testIf(
+      () => process.platform !== `win32`,
       `it should generate a file that can be used as an executable to resolve a request (builtin request)`,
       makeTemporaryEnv(
         {
@@ -797,7 +799,8 @@ module.exports = makeTemporaryEnv => {
       ),
     );
 
-    test(
+    testIf(
+      () => process.platform !== `win32`,
       `it should generate a file that can be used as an executable to resolve a request (invalid request)`,
       makeTemporaryEnv(
         {

packages/pkg-tests/pkg-tests-specs/sources/pnpapi-v1.js

@@ -45,9 +45,9 @@ module.exports = makeTemporaryEnv => {
         makeTemporaryEnv({}, {plugNPlay: true}, async ({path, run, source}) => {
           await run(`install`);
 
-          await expect(source(`require('pnpapi').resolveRequest('pnpapi', '${path}/')`)).resolves.toEqual(
-            normalize(`${path}/.pnp.js`),
-          );
+          await expect(
+            source(`require('pnpapi').resolveRequest('pnpapi', ${JSON.stringify(path + '/')})`),
+          ).resolves.toEqual(normalize(`${path}/.pnp.js`));
         }),
       );
 
@@ -56,7 +56,9 @@ module.exports = makeTemporaryEnv => {
         makeTemporaryEnv({}, {plugNPlay: true}, async ({path, run, source}) => {
           await run(`install`);
 
-          await expect(source(`require('pnpapi').resolveRequest('fs', '${path}/')`)).resolves.toEqual(null);
+          await expect(
+            source(`require('pnpapi').resolveRequest('fs', ${JSON.stringify(path)} + '/')`),
+          ).resolves.toEqual(null);
         }),
       );
 
@@ -77,8 +79,10 @@ module.exports = makeTemporaryEnv => {
             await run(`install`);
 
             await expect(
-              source(`require('pnpapi').resolveRequest('fs', '${path}/', {considerBuiltins: false})`),
-            ).resolves.toEqual(`${path}/fs/index.js`);
+              source(
+                `require('pnpapi').resolveRequest('fs', ${JSON.stringify(path)} + '/', {considerBuiltins: false})`,
+              ),
+            ).resolves.toEqual(normalize(`${path}/fs/index.js`));
           },
         ),
       );
@@ -91,8 +95,8 @@ module.exports = makeTemporaryEnv => {
           await run(`install`);
 
           await expect(
-            source(`require('pnpapi').resolveRequest('./foo', '${path}/', {extensions: ['.bar']})`),
-          ).resolves.toEqual(`${path}/foo.bar`);
+            source(`require('pnpapi').resolveRequest('./foo', ${JSON.stringify(path)} + '/', {extensions: ['.bar']})`),
+          ).resolves.toEqual(normalize(`${path}/foo.bar`));
         }),
       );
     });

packages/pkg-tests/pkg-tests-specs/sources/script.js

@@ -12,22 +12,18 @@ module.exports = (makeTemporaryEnv: PackageDriver) => {
     test(
       `it should run scripts using the same Node than the one used by Yarn`,
       makeTemporaryEnv({scripts: {myScript: `node --version`}}, async ({path, run, source}) => {
-        await makeFakeBinary(`${path}/bin/node`);
+        await makeFakeBinary(`${path}/bin/node`, {exitCode: 1});
 
-        await expect(run(`run`, `myScript`)).resolves.toMatchObject({
-          stdout: `${process.version}\n`,
-        });
+        await expect(run(`run`, `myScript`)).resolves.toBeTruthy();
       }),
     );
 
     test(
       `it should run scripts using the same package manager than the one running the scripts`,
       makeTemporaryEnv({scripts: {myScript: `yarn --version`}}, async ({path, run, source}) => {
-        await makeFakeBinary(`${path}/bin/yarn`);
+        await makeFakeBinary(`${path}/bin/yarn`, {exitCode: 1});
 
-        await expect(run(`run`, `myScript`)).resolves.toMatchObject({
-          stdout: (await run(`--version`)).stdout,
-        });
+        await expect(run(`run`, `myScript`)).resolves.toBeTruthy();
       }),
     );
 
@@ -200,7 +196,11 @@ module.exports = (makeTemporaryEnv: PackageDriver) => {
       makeTemporaryEnv({dependencies: {[`no-deps-scripted`]: `1.0.0`}}, async ({path, run, source}) => {
         await run(`install`);
 
-        await expect(source(`require('no-deps-scripted/log.js')`)).resolves.toEqual([100, 200, 300]);
+        await expect(source(`require('no-deps-scripted/log.js')`)).resolves.toEqual([
+          'preinstall',
+          'install',
+          'postinstall',
+        ]);
       }),
     );
   });