From e8794dece21f20cc6c809f507e342030c5d0d2d4 Mon Sep 17 00:00:00 2001 From: Ofek Atar Date: Thu, 22 Sep 2022 13:55:44 +0300 Subject: [PATCH] chore: Compute analytics for experimental IaC test --- src/lib/iac/test/v2/analytics/iac-type.ts | 123 ++++++++++++++++++ src/lib/iac/test/v2/analytics/index.ts | 46 +++++++ src/lib/iac/test/v2/index.ts | 7 +- src/lib/iac/test/v2/scan/results.ts | 6 +- .../v2/analytics/fixtures/iac-analytics.json | 21 +++ .../lib/iac/test/v2/analytics/index.spec.ts | 58 +++++++++ 6 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 src/lib/iac/test/v2/analytics/iac-type.ts create mode 100644 src/lib/iac/test/v2/analytics/index.ts create mode 100644 test/jest/unit/lib/iac/test/v2/analytics/fixtures/iac-analytics.json create mode 100644 test/jest/unit/lib/iac/test/v2/analytics/index.spec.ts diff --git a/src/lib/iac/test/v2/analytics/iac-type.ts b/src/lib/iac/test/v2/analytics/iac-type.ts new file mode 100644 index 0000000000..3d0b94411e --- /dev/null +++ b/src/lib/iac/test/v2/analytics/iac-type.ts @@ -0,0 +1,123 @@ +import { SEVERITY } from '../../../../snyk-test/legacy'; +import { ResourceKind, TestOutput } from '../scan/results'; + +export function getIacType(testOutput: TestOutput): IacType { + const resourcesCountByPackageManager = getResourcesCountByPackageManager( + testOutput, + ); + + const filesCountByPackageManager = getFilesCountByPackageManager(testOutput); + + const vulnAnalyticsByPackageManager = getVulnerabilityAnalyticsByPackageManager( + testOutput, + ); + + return Object.keys(resourcesCountByPackageManager).reduce( + (acc, packageManager) => { + acc[packageManager] = { + count: filesCountByPackageManager[packageManager], + 'resource-count': resourcesCountByPackageManager[packageManager], + ...vulnAnalyticsByPackageManager[packageManager], + }; + return acc; + }, + {}, + ); +} + +export type PackageManager = ResourceKind; + +export type IacType = { + [packageManager in PackageManager]?: { + count: number; + 'resource-count': number; + } & { + [severity in SEVERITY]?: number; + }; +}; + +function getResourcesCountByPackageManager( + testOutput: TestOutput, +): ResourcesCountByPackageManager { + if (!testOutput.results?.resources?.length) { + return {}; + } + + return testOutput.results.resources.reduce((acc, resource) => { + const packageManager = resource.kind; + + if (!acc[packageManager]) { + acc[packageManager] = 0; + } + + acc[packageManager]++; + + return acc; + }, {}); +} + +export type ResourcesCountByPackageManager = { + [packageManager in PackageManager]?: number; +}; + +function getFilesCountByPackageManager( + testOutput: TestOutput, +): FilesCountByPackageManager { + if (!testOutput.results?.resources?.length) { + return {}; + } + + return Object.entries( + testOutput.results.resources.reduce((acc, resource) => { + const packageManager = resource.kind; + + if (!acc[packageManager]) { + acc[packageManager] = new Set(); + } + + acc[packageManager].add(resource.file); + + return acc; + }, {} as { [packageManager in PackageManager]: Set }), + ).reduce((acc, [packageManager, filesSet]) => { + acc[packageManager] = filesSet.size; + + return acc; + }, {}); +} + +export type FilesCountByPackageManager = { + [packageManager in PackageManager]?: number; +}; + +function getVulnerabilityAnalyticsByPackageManager( + testOutput: TestOutput, +): VulnerabilityAnalyticsByPackageManager { + if (!testOutput.results?.vulnerabilities?.length) { + return {}; + } + + return testOutput.results.vulnerabilities.reduce((acc, vuln) => { + const packageManager = vuln.resource.kind; + + if (!acc[packageManager]) { + acc[packageManager] = {}; + } + + if (!acc[packageManager][vuln.severity]) { + acc[packageManager][vuln.severity] = 0; + } + + acc[packageManager][vuln.severity]++; + + return acc; + }, {}); +} + +export type VulnerabilityAnalyticsByPackageManager = { + [packageManager in PackageManager]?: VulnerabilityAnalitycs; +}; + +export type VulnerabilityAnalitycs = { + [severity in SEVERITY]?: number; +}; diff --git a/src/lib/iac/test/v2/analytics/index.ts b/src/lib/iac/test/v2/analytics/index.ts new file mode 100644 index 0000000000..2528d93b31 --- /dev/null +++ b/src/lib/iac/test/v2/analytics/index.ts @@ -0,0 +1,46 @@ +import * as createDebugLogger from 'debug'; + +import { policyEngineReleaseVersion } from '../local-cache/policy-engine/constants'; +import { ResourceKind, TestOutput } from '../scan/results'; +import { getIacType, IacType } from './iac-type'; + +const debugLog = createDebugLogger('snyk-iac'); + +export interface IacAnalytics { + packageManager: ResourceKind[]; + 'iac-type': IacType; + 'iac-issues-count': number; + 'iac-ignored-issues-count': number; + 'iac-files-count': number; + 'iac-resources-count': number; + 'iac-test-binary-version': string; + 'iac-error-codes': number[]; + // 'iac-rules-bundle-version': string; // TODO: Add when we have the rules bundle version +} + +export function addIacAnalytics(testOutput: TestOutput): void { + const iacAnalytics = computeIacAnalytics(testOutput); + + debugLog(iacAnalytics); +} + +export function computeIacAnalytics(testOutput: TestOutput): IacAnalytics { + const iacType = getIacType(testOutput); + + return { + 'iac-type': iacType, + packageManager: Object.keys(iacType) as ResourceKind[], + 'iac-issues-count': testOutput.results?.vulnerabilities?.length || 0, + 'iac-ignored-issues-count': + testOutput.results?.scanAnalytics.ignoredCount || 0, + 'iac-files-count': Object.values(iacType).reduce( + (acc, packageManagerAnalytics) => acc + packageManagerAnalytics!.count, + 0, + ), + 'iac-resources-count': testOutput.results?.resources?.length || 0, + 'iac-error-codes': + [...new Set(testOutput.errors?.map((error) => error.code!))] || [], + 'iac-test-binary-version': policyEngineReleaseVersion, + // iacAnalytics['iac-rules-bundle-version'] = ''; // TODO: Add when we have the rules bundle version + }; +} diff --git a/src/lib/iac/test/v2/index.ts b/src/lib/iac/test/v2/index.ts index 0f16c68241..939c7ffd8e 100644 --- a/src/lib/iac/test/v2/index.ts +++ b/src/lib/iac/test/v2/index.ts @@ -2,6 +2,7 @@ import { TestConfig } from './types'; import { scan } from './scan'; import { TestOutput } from './scan/results'; import { initLocalCache } from './local-cache'; +import { addIacAnalytics } from './analytics'; export { TestConfig } from './types'; @@ -10,5 +11,9 @@ export async function test(testConfig: TestConfig): Promise { testConfig, ); - return scan(testConfig, policyEnginePath, rulesBundlePath); + const testOutput = scan(testConfig, policyEnginePath, rulesBundlePath); + + addIacAnalytics(testOutput); + + return testOutput; } diff --git a/src/lib/iac/test/v2/scan/results.ts b/src/lib/iac/test/v2/scan/results.ts index fee2a75225..fb4531f65b 100644 --- a/src/lib/iac/test/v2/scan/results.ts +++ b/src/lib/iac/test/v2/scan/results.ts @@ -71,11 +71,15 @@ export interface Resource { path?: any[]; formattedPath: string; file: string; - kind: IacProjectType | PolicyEngineTypes.State.InputTypeEnum; + kind: ResourceKind; line?: number; column?: number; } +export type ResourceKind = + | IacProjectType + | PolicyEngineTypes.State.InputTypeEnum; + export interface ScanError { message: string; code: number; diff --git a/test/jest/unit/lib/iac/test/v2/analytics/fixtures/iac-analytics.json b/test/jest/unit/lib/iac/test/v2/analytics/fixtures/iac-analytics.json new file mode 100644 index 0000000000..402e567570 --- /dev/null +++ b/test/jest/unit/lib/iac/test/v2/analytics/fixtures/iac-analytics.json @@ -0,0 +1,21 @@ +{ + "iac-type": { + "terraformconfig": { + "count": 2, + "resource-count": 4, + "medium": 4 + } + }, + "packageManager": [ + "terraformconfig" + ], + "iac-issues-count": 4, + "iac-ignored-issues-count": 3, + "iac-files-count": 2, + "iac-error-codes": [ + 2114 + ], + "iac-resources-count": 4, + "iac-test-binary-version": "test-policy-engine-release-version" + } + \ No newline at end of file diff --git a/test/jest/unit/lib/iac/test/v2/analytics/index.spec.ts b/test/jest/unit/lib/iac/test/v2/analytics/index.spec.ts new file mode 100644 index 0000000000..01a20ddf79 --- /dev/null +++ b/test/jest/unit/lib/iac/test/v2/analytics/index.spec.ts @@ -0,0 +1,58 @@ +import * as clonedeep from 'lodash.clonedeep'; +import * as path from 'path'; +import * as fs from 'fs'; + +import { SnykIacTestOutput } from '../../../../../../../../src/lib/iac/test/v2/scan/results'; +import { + computeIacAnalytics, + IacAnalytics, +} from '../../../../../../../../src/lib/iac/test/v2/analytics'; + +jest.mock( + '../../../../../../../../src/lib/iac/test/v2/local-cache/policy-engine/constants', + () => ({ + ...jest.requireActual( + '../../../../../../../../src/lib/iac/test/v2/local-cache/policy-engine/constants', + ), + policyEngineReleaseVersion: 'test-policy-engine-release-version', + }), +); + +describe('computeIacAnalytics', () => { + const snykIacTestOutputFixture: SnykIacTestOutput = JSON.parse( + fs.readFileSync( + path.join( + __dirname, + '..', + '..', + '..', + '..', + '..', + 'iac', + 'process-results', + 'fixtures', + 'snyk-iac-test-results.json', + ), + 'utf-8', + ), + ); + + const iacAnalyticsFixture: IacAnalytics = JSON.parse( + fs.readFileSync( + path.join(__dirname, 'fixtures', 'iac-analytics.json'), + 'utf-8', + ), + ); + + it('generates the correct analytics', async () => { + // Arrange + const testOutput = clonedeep(snykIacTestOutputFixture); + const expectedAnalytics = clonedeep(iacAnalyticsFixture); + + // Act + const result = computeIacAnalytics(testOutput); + + // Assert + expect(result).toStrictEqual(expectedAnalytics); + }); +});