feat: Vulns to use exit code 1, all other errors 2

src/cli/commands/test.ts

@@ -103,6 +103,11 @@ async function test(...args): Promise<string> {
     }
   }
 
+  const vulnerableResults = results.filter((res) => res.vulnerabilities && res.vulnerabilities.length);
+  const errorResults = results.filter((res) => res instanceof Error);
+  const notSuccess = errorResults.length > 0;
+  const foundVulnerabilities = vulnerableResults.length > 0;
+
   // resultOptions is now an array of 1 or more options used for
   // the tests results is now an array of 1 or more test results
   // values depend on `options.json` value - string or object
@@ -121,22 +126,27 @@ async function test(...args): Promise<string> {
 
     // backwards compat - strip array IFF only one result
     const dataToSend = results.length === 1 ? results[0] : results;
-    const json = JSON.stringify(dataToSend, null, 2);
+    const stringifiedError = JSON.stringify(dataToSend, null, 2);
 
     if (results.every((res) => res.ok)) {
-      return json;
+      return stringifiedError;
+    }
+    const err = new Error(stringifiedError) as any;
+    if (foundVulnerabilities) {
+      err.code = 'VULNS';
+      const dataToSendNoVulns = dataToSend;
+      delete dataToSendNoVulns.vulnerabilities;
+      err.jsonNoVulns = dataToSendNoVulns;
     }
 
-    throw new Error(json);
+    err.json = stringifiedError;
+    throw err;
   }
 
   let response = results.map((unused, i) => displayResult(results[i], resultOptions[i]))
     .join(`\n${SEPARATOR}`);
 
-  const vulnerableResults = results.filter((res) => res.vulnerabilities && res.vulnerabilities.length);
-  const errorResults = results.filter((res) => res instanceof Error);
-
-  if (errorResults.length > 0) {
+  if (notSuccess) {
     debug(`Failed to test ${errorResults.length} projects, errors:`);
     errorResults.forEach((err) => {
       const errString = err.stack ? err.stack.toString() : err.toString();
@@ -153,17 +163,27 @@ async function test(...args): Promise<string> {
       summariseErrorResults(errorResults) + '\n';
   }
 
-  const notSuccess = vulnerableResults.length > 0 || errorResults.length > 0;
-
   if (notSuccess) {
     response += chalk.bold.red(summaryMessage);
     const error = new Error(response) as any;
     // take the code of the first problem to go through error
     // translation
     // HACK as there can be different errors, and we pass only the
     // first one
-    error.code = (vulnerableResults[0] || errorResults[0]).code;
-    error.userMessage = (vulnerableResults[0] || errorResults[0]).userMessage;
+    error.code = errorResults[0].code;
+    error.userMessage = errorResults[0].userMessage;
+    throw error;
+  }
+
+  if (foundVulnerabilities) {
+    response += chalk.bold.red(summaryMessage);
+    const error = new Error(response) as any;
+    // take the code of the first problem to go through error
+    // translation
+    // HACK as there can be different errors, and we pass only the
+    // first one
+    error.code = vulnerableResults[0].code || 'VULNS';
+    error.userMessage = vulnerableResults[0].userMessage;
     throw error;
   }
 

src/cli/index.ts

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 import 'source-map-support/register';
+import * as Debug from 'debug';
 
 // assert supported node runtime version
 import * as runtime from './runtime';
@@ -16,9 +17,14 @@ import {isPathToPackageFile} from '../lib/detect';
 import {updateCheck} from '../lib/updater';
 import { MissingTargetFileError } from '../lib/errors/missing-targetfile-error';
 
+const debug = Debug('snyk');
+const EXIT_CODES = {
+  VULNS_FOUND: 1,
+  ERROR: 2,
+};
+
 async function runCommand(args) {
   const result = await args.method(...args.options._);
-
   const res = analytics({
     args: args.options._,
     command: args.command,
@@ -39,38 +45,22 @@ async function runCommand(args) {
 async function handleError(args, error) {
   spinner.clearAll();
   let command = 'bad-command';
+  let exitCode = EXIT_CODES.ERROR;
 
-  if (error.code === 'VULNS') {
+  const vulnsFound = (error.code === 'VULNS');
+  if (vulnsFound) {
     // this isn't a bad command, so we won't record it as such
     command = args.command;
-  } else if (!error.stack) { // log errors that are not error objects
-    analytics.add('error', JSON.stringify(error));
-    analytics.add('command', args.command);
-  } else {
-    // remove vulnerabilities from the errors
-    // to keep the logs small
-    if (error.stack && error.stack.vulnerabilities) {
-      delete error.vulnerabilities;
-    }
-    if (error.message && error.message.vulnerabilities) {
-      delete error.message.vulnerabilities;
-    }
-    analytics.add('error-message', error.message);
-    analytics.add('error', error.stack);
-    analytics.add('error-code', error.code);
-    analytics.add('command', args.command);
+    exitCode = EXIT_CODES.VULNS_FOUND;
   }
 
-  const res = analytics({
-    args: args.options._,
-    command,
-  });
-
-  if (args.options.debug) {
-    console.log(error.stack);
+  if (args.options.debug && !args.options.json) {
+    const output = vulnsFound ? error.message : error.stack;
+    console.log(output);
+  } else if (args.options.json) {
+    console.log(error.json || error.stack);
   } else {
     if (!args.options.quiet) {
-
       const result = errors.message(error);
       if (args.options.copy) {
         copy(result);
@@ -86,15 +76,40 @@ async function handleError(args, error) {
     }
   }
 
-  return res;
+  const analyticsError = vulnsFound ? {
+    stack: error.jsonNoVulns,
+    code: error.code,
+    message: 'Vulnerabilities found',
+   } : {
+    stack: error.stack,
+    code: error.code,
+    message: error.message,
+   };
+
+  if (!vulnsFound && !error.stack) { // log errors that are not error objects
+    analytics.add('error', JSON.stringify(analyticsError));
+    analytics.add('command', args.command);
+  } else {
+    analytics.add('error-message', analyticsError.message);
+    analytics.add('error', analyticsError.stack);
+    analytics.add('error-code', error.code);
+    analytics.add('command', args.command);
+  }
+
+  const res = analytics({
+    args: args.options._,
+    command,
+  });
+
+  return { res, exitCode };
 }
 
 function checkRuntime() {
   if (!runtime.isSupported(process.versions.node)) {
     console.error(`${process.versions.node} is an unsupported nodejs ` +
       `runtime! Supported runtime range is '${runtime.supportedRange}'`);
     console.error('Please upgrade your nodejs runtime version and try again.');
-    process.exit(1);
+    process.exit(EXIT_CODES.ERROR);
   }
 }
 
@@ -113,9 +128,9 @@ async function main() {
   checkRuntime();
 
   const args = argsLib(process.argv);
-  let res = null;
+  let res;
   let failed = false;
-
+  let exitCode = EXIT_CODES.ERROR;
   try {
     if (args.options.file && args.options.file.match(/\.sln$/)) {
       sln.updateArgs(args);
@@ -124,23 +139,28 @@ async function main() {
     res = await runCommand(args);
   } catch (error) {
     failed = true;
-    res = await handleError(args, error);
+
+    const response = await handleError(args, error);
+    res = response.res;
+    exitCode = response.exitCode;
   }
 
   if (!args.options.json) {
     console.log(alerts.displayAlerts());
   }
 
   if (!process.env.TAP && failed) {
-    process.exit(1);
+    debug('Exit code: ' + exitCode);
+    process.exitCode = exitCode;
   }
 
   return res;
 }
 
 const cli = main().catch((e) => {
-  console.log('super fail', e.stack);
-  process.exit(1);
+  console.error('Something unexpected went wrong: ', e.stack);
+  console.error('Exit code: ' + EXIT_CODES.ERROR);
+  process.exit(EXIT_CODES.ERROR);
 });
 
 if (module.parent) {

src/lib/analytics.js

@@ -1,20 +1,20 @@
 module.exports = analytics;
 module.exports.single = postAnalytics;
 
-var snyk = require('../lib');
-var config = require('./config');
-var version = require('./version');
-var request = require('./request');
-var isCI = require('./is-ci').isCI;
-var debug = require('debug')('snyk');
-var os = require('os');
-var osName = require('os-name');
-var crypto = require('crypto');
-var uuid = require('uuid');
+const snyk = require('../lib');
+const config = require('./config');
+const version = require('./version');
+const request = require('./request');
+const isCI = require('./is-ci').isCI;
+const debug = require('debug')('snyk');
+const os = require('os');
+const osName = require('os-name');
+const crypto = require('crypto');
+const uuid = require('uuid');
 
-var metadata = {};
+const metadata = {};
 // analytics module is required at the beginning of the CLI run cycle
-var startTime = Date.now();
+const startTime = Date.now();
 
 function analytics(data) {
   if (!data) {
@@ -48,11 +48,11 @@ function postAnalytics(data) {
     data.os = osName(os.platform(), os.release());
     data.nodeVersion = process.version;
 
-    var seed = uuid.v4();
-    var shasum = crypto.createHash('sha1');
+    const seed = uuid.v4();
+    const shasum = crypto.createHash('sha1');
     data.id = shasum.update(seed).digest('hex');
 
-    var headers = {};
+    const headers = {};
     if (snyk.api) {
       headers.authorization = 'token ' + snyk.api;
     }
@@ -76,12 +76,8 @@ function postAnalytics(data) {
   });
 }
 
-analytics.reset = function () {
-  metadata = {};
-};
-
 analytics.add = function (key, value) {
-  debug('analytics add', key, value);
+  debug('analytics adding to metadata: ', key, value);
   if (metadata[key]) {
     if (!Array.isArray(metadata[key])) {
       metadata[key] = [metadata[key]];