Merge pull request #3857 from snyk/chore/add-experimental-test-analyt…

…ics-cfg-2157

chore: Add experimental test analytics [CFG-2157]

src/lib/formatters/iac-output/text/formatters.ts

@@ -106,7 +106,7 @@ export function formatSnykIacTestTestData(
   const filesWithIssuesCount = countFilesWithIssues(snykIacTestScanResult);
   const filesWithoutIssuesCount = allFilesCount - filesWithIssuesCount;
   const ignores = snykIacTestScanResult
-    ? snykIacTestScanResult.metadata.ignoredCount
+    ? snykIacTestScanResult.scanAnalytics.ignoredCount
     : 0;
 
   let contextSuppressedIssueCount: number | undefined;

src/lib/iac/test/v2/analytics/iac-type.ts

@@ -0,0 +1,123 @@
+import { SEVERITY } from '../../../../snyk-test/legacy';
+import { ResourceKind, TestOutput } from '../scan/results';
+
+export function getIacType(testOutput: TestOutput): IacType {
+  const resourcesCountByPackageManager = getResourcesCountByPackageManager(
+    testOutput,
+  );
+
+  const filesCountByPackageManager = getFilesCountByPackageManager(testOutput);
+
+  const vulnAnalyticsByPackageManager = getVulnerabilityAnalyticsByPackageManager(
+    testOutput,
+  );
+
+  return Object.keys(resourcesCountByPackageManager).reduce(
+    (acc, packageManager) => {
+      acc[packageManager] = {
+        count: filesCountByPackageManager[packageManager],
+        'resource-count': resourcesCountByPackageManager[packageManager],
+        ...vulnAnalyticsByPackageManager[packageManager],
+      };
+      return acc;
+    },
+    {},
+  );
+}
+
+export type PackageManager = ResourceKind;
+
+export type IacType = {
+  [packageManager in PackageManager]?: {
+    count: number;
+    'resource-count': number;
+  } & {
+    [severity in SEVERITY]?: number;
+  };
+};
+
+function getResourcesCountByPackageManager(
+  testOutput: TestOutput,
+): ResourcesCountByPackageManager {
+  if (!testOutput.results?.resources?.length) {
+    return {};
+  }
+
+  return testOutput.results.resources.reduce((acc, resource) => {
+    const packageManager = resource.kind;
+
+    if (!acc[packageManager]) {
+      acc[packageManager] = 0;
+    }
+
+    acc[packageManager]++;
+
+    return acc;
+  }, {});
+}
+
+export type ResourcesCountByPackageManager = {
+  [packageManager in PackageManager]?: number;
+};
+
+function getFilesCountByPackageManager(
+  testOutput: TestOutput,
+): FilesCountByPackageManager {
+  if (!testOutput.results?.resources?.length) {
+    return {};
+  }
+
+  return Object.entries(
+    testOutput.results.resources.reduce((acc, resource) => {
+      const packageManager = resource.kind;
+
+      if (!acc[packageManager]) {
+        acc[packageManager] = new Set();
+      }
+
+      acc[packageManager].add(resource.file);
+
+      return acc;
+    }, {} as { [packageManager in PackageManager]: Set<string> }),
+  ).reduce((acc, [packageManager, filesSet]) => {
+    acc[packageManager] = filesSet.size;
+
+    return acc;
+  }, {});
+}
+
+export type FilesCountByPackageManager = {
+  [packageManager in PackageManager]?: number;
+};
+
+function getVulnerabilityAnalyticsByPackageManager(
+  testOutput: TestOutput,
+): VulnerabilityAnalyticsByPackageManager {
+  if (!testOutput.results?.vulnerabilities?.length) {
+    return {};
+  }
+
+  return testOutput.results.vulnerabilities.reduce((acc, vuln) => {
+    const packageManager = vuln.resource.kind;
+
+    if (!acc[packageManager]) {
+      acc[packageManager] = {};
+    }
+
+    if (!acc[packageManager][vuln.severity]) {
+      acc[packageManager][vuln.severity] = 0;
+    }
+
+    acc[packageManager][vuln.severity]++;
+
+    return acc;
+  }, {});
+}
+
+export type VulnerabilityAnalyticsByPackageManager = {
+  [packageManager in PackageManager]?: VulnerabilityAnalitycs;
+};
+
+export type VulnerabilityAnalitycs = {
+  [severity in SEVERITY]?: number;
+};

src/lib/iac/test/v2/analytics/index.ts

@@ -0,0 +1,46 @@
+import * as createDebugLogger from 'debug';
+
+import { policyEngineReleaseVersion } from '../local-cache/policy-engine/constants';
+import { ResourceKind, TestOutput } from '../scan/results';
+import { getIacType, IacType } from './iac-type';
+
+const debugLog = createDebugLogger('snyk-iac');
+
+export interface IacAnalytics {
+  packageManager: ResourceKind[];
+  'iac-type': IacType;
+  'iac-issues-count': number;
+  'iac-ignored-issues-count': number;
+  'iac-files-count': number;
+  'iac-resources-count': number;
+  'iac-test-binary-version': string;
+  'iac-error-codes': number[];
+  // 'iac-rules-bundle-version': string; // TODO: Add when we have the rules bundle version
+}
+
+export function addIacAnalytics(testOutput: TestOutput): void {
+  const iacAnalytics = computeIacAnalytics(testOutput);
+
+  debugLog(iacAnalytics);
+}
+
+export function computeIacAnalytics(testOutput: TestOutput): IacAnalytics {
+  const iacType = getIacType(testOutput);
+
+  return {
+    'iac-type': iacType,
+    packageManager: Object.keys(iacType) as ResourceKind[],
+    'iac-issues-count': testOutput.results?.vulnerabilities?.length || 0,
+    'iac-ignored-issues-count':
+      testOutput.results?.scanAnalytics.ignoredCount || 0,
+    'iac-files-count': Object.values(iacType).reduce(
+      (acc, packageManagerAnalytics) => acc + packageManagerAnalytics!.count,
+      0,
+    ),
+    'iac-resources-count': testOutput.results?.resources?.length || 0,
+    'iac-error-codes':
+      [...new Set(testOutput.errors?.map((error) => error.code!))] || [],
+    'iac-test-binary-version': policyEngineReleaseVersion,
+    // iacAnalytics['iac-rules-bundle-version'] = ''; // TODO: Add when we have the rules bundle version
+  };
+}

src/lib/iac/test/v2/index.ts

@@ -2,6 +2,7 @@ import { TestConfig } from './types';
 import { scan } from './scan';
 import { TestOutput } from './scan/results';
 import { initLocalCache } from './local-cache';
+import { addIacAnalytics } from './analytics';
 
 export { TestConfig } from './types';
 
@@ -10,5 +11,9 @@ export async function test(testConfig: TestConfig): Promise<TestOutput> {
     testConfig,
   );
 
-  return scan(testConfig, policyEnginePath, rulesBundlePath);
+  const testOutput = scan(testConfig, policyEnginePath, rulesBundlePath);
+
+  addIacAnalytics(testOutput);
+
+  return testOutput;
 }

src/lib/iac/test/v2/scan/results.ts

@@ -39,11 +39,11 @@ export interface Results {
 
 export interface Metadata {
   projectName: string;
-  ignoredCount: number;
 }
 
 export interface ScanAnalytics {
   suppressedResults?: Record<string, string[]>;
+  ignoredCount: number;
 }
 
 export interface Vulnerability {
@@ -71,11 +71,15 @@ export interface Resource {
   path?: any[];
   formattedPath: string;
   file: string;
-  kind: IacProjectType | PolicyEngineTypes.State.InputTypeEnum;
+  kind: ResourceKind;
   line?: number;
   column?: number;
 }
 
+export type ResourceKind =
+  | IacProjectType
+  | PolicyEngineTypes.State.InputTypeEnum;
+
 export interface ScanError {
   message: string;
   code: number;

test/jest/unit/iac/process-results/fixtures/snyk-iac-test-results-with-suppressions.json

@@ -147,14 +147,14 @@
       }
     ],
     "metadata": {
-      "projectName": "input-files-for-json-v2",
-      "ignoredCount": 3
+      "projectName": "input-files-for-json-v2"
     },
     "scanAnalytics": {
       "suppressedResults": {
         "issue-1": ["resource1", "resource2"],
         "issue-2": ["resource3"]
-      }
+      },
+      "ignoredCount": 3
     }
   },
   "errors": [

test/jest/unit/iac/process-results/fixtures/snyk-iac-test-results.json

@@ -147,10 +147,11 @@
       }
     ],
     "metadata": {
-      "projectName": "input-files-for-json-v2",
-      "ignoredCount": 3
+      "projectName": "input-files-for-json-v2"
     },
-    "scanAnalytics": {}
+    "scanAnalytics": {
+      "ignoredCount": 3
+    }
   },
   "errors": [
     {

test/jest/unit/lib/iac/test/v2/analytics/fixtures/iac-analytics.json

@@ -0,0 +1,21 @@
+{
+    "iac-type": {
+      "terraformconfig": {
+        "count": 2,
+        "resource-count": 4,
+        "medium": 4
+      }
+    },
+    "packageManager": [
+      "terraformconfig"
+    ],
+    "iac-issues-count": 4,
+    "iac-ignored-issues-count": 3,
+    "iac-files-count": 2,
+    "iac-error-codes": [
+     2114
+    ],
+    "iac-resources-count": 4,
+    "iac-test-binary-version": "test-policy-engine-release-version"
+  }
+

test/jest/unit/lib/iac/test/v2/analytics/index.spec.ts

@@ -0,0 +1,58 @@
+import * as clonedeep from 'lodash.clonedeep';
+import * as path from 'path';
+import * as fs from 'fs';
+
+import { SnykIacTestOutput } from '../../../../../../../../src/lib/iac/test/v2/scan/results';
+import {
+  computeIacAnalytics,
+  IacAnalytics,
+} from '../../../../../../../../src/lib/iac/test/v2/analytics';
+
+jest.mock(
+  '../../../../../../../../src/lib/iac/test/v2/local-cache/policy-engine/constants',
+  () => ({
+    ...jest.requireActual(
+      '../../../../../../../../src/lib/iac/test/v2/local-cache/policy-engine/constants',
+    ),
+    policyEngineReleaseVersion: 'test-policy-engine-release-version',
+  }),
+);
+
+describe('computeIacAnalytics', () => {
+  const snykIacTestOutputFixture: SnykIacTestOutput = JSON.parse(
+    fs.readFileSync(
+      path.join(
+        __dirname,
+        '..',
+        '..',
+        '..',
+        '..',
+        '..',
+        'iac',
+        'process-results',
+        'fixtures',
+        'snyk-iac-test-results.json',
+      ),
+      'utf-8',
+    ),
+  );
+
+  const iacAnalyticsFixture: IacAnalytics = JSON.parse(
+    fs.readFileSync(
+      path.join(__dirname, 'fixtures', 'iac-analytics.json'),
+      'utf-8',
+    ),
+  );
+
+  it('generates the correct analytics', async () => {
+    // Arrange
+    const testOutput = clonedeep(snykIacTestOutputFixture);
+    const expectedAnalytics = clonedeep(iacAnalyticsFixture);
+
+    // Act
+    const result = computeIacAnalytics(testOutput);
+
+    // Assert
+    expect(result).toStrictEqual(expectedAnalytics);
+  });
+});