From fdb7f18fe920e227f18fd2ae5e5d9cf6fcea62e1 Mon Sep 17 00:00:00 2001 From: dblock Date: Thu, 15 Aug 2024 14:19:11 -0400 Subject: [PATCH] Refactored paths into operations. Signed-off-by: dblock --- .../pr-test-coverage-analysis.template.md | 6 +-- TESTING_GUIDE.md | 4 +- tools/src/tester/ChapterEvaluator.ts | 4 ++ tools/src/tester/ResultLogger.ts | 32 +++++------ tools/src/tester/StoryEvaluator.ts | 4 +- tools/src/tester/TestResults.ts | 54 +++++++++++-------- tools/src/tester/types/eval.types.ts | 6 +++ tools/src/tester/types/test.types.ts | 4 +- tools/tests/tester/ResultLogger.test.ts | 12 ++++- tools/tests/tester/TestResults.test.ts | 46 +++++++++------- 10 files changed, 102 insertions(+), 70 deletions(-) diff --git a/.github/pr-comment-templates/pr-test-coverage-analysis.template.md b/.github/pr-comment-templates/pr-test-coverage-analysis.template.md index 27f0fe7b5..212dd595b 100644 --- a/.github/pr-comment-templates/pr-test-coverage-analysis.template.md +++ b/.github/pr-comment-templates/pr-test-coverage-analysis.template.md @@ -1,8 +1,8 @@ ## Spec Test Coverage Analysis {{with .test_coverage}} -| Total | Tested | -|-------------------|----------------------------------------------------------| -| {{.paths_count}} | {{.evaluated_paths_count}} ({{.evaluated_paths_pct}} %) | +| Total | Tested | +|------------------------------|---------------------------------------------------------------| +| {{.total_operations_count}} | {{.evaluated_operations_count}} ({{.evaluated_paths_pct}} %) | {{end}} \ No newline at end of file diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 2dd25fd3a..251f9c9a4 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -330,8 +330,8 @@ The test tool can generate a test coverage summary using `--coverage ` wit ```json { - "evaluated_paths_count": 214, - "paths_count": 550, + "evaluated_operations_count": 214, + "operations_count": 550, "evaluated_paths_pct": 38.91 } ``` diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index dee177ddc..6feb7d0a0 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -84,6 +84,10 @@ export default class ChapterEvaluator { var result: ChapterEvaluation = { title: chapter.synopsis, + operation: { + method: chapter.method, + path: chapter.path + }, path: `${chapter.method} ${chapter.path}`, overall: { result: overall_result(evaluations) }, request: { parameters: params, request }, diff --git a/tools/src/tester/ResultLogger.ts b/tools/src/tester/ResultLogger.ts index 45cd601c6..0d6d2c929 100644 --- a/tools/src/tester/ResultLogger.ts +++ b/tools/src/tester/ResultLogger.ts @@ -7,7 +7,7 @@ * compatible open source license. */ -import { type ChapterEvaluation, type Evaluation, Result, type StoryEvaluation } from './types/eval.types' +import { type ChapterEvaluation, type Evaluation, Operation, Result, type StoryEvaluation } from './types/eval.types' import { overall_result } from './helpers' import * as ansi from './Ansi' import TestResults from './TestResults' @@ -44,29 +44,29 @@ export class ConsoleResultLogger implements ResultLogger { log_coverage(results: TestResults): void { console.log() - console.log(`Tested ${results.evaluated_paths().length}/${results.spec_paths().length} paths.`) + console.log(`Tested ${results.evaluated_operations().length}/${results.operations().length} paths.`) } log_coverage_report(results: TestResults): void { console.log() - console.log(`${results.unevaluated_paths().length} paths remaining.`) - const groups = _.groupBy(results.unevaluated_paths(), (path) => path.split(' ', 2)[1].split('/')[1]) - Object.entries(groups).forEach(([root, paths]) => { - this.#log_coverage_group(root, paths) + console.log(`${results.unevaluated_operations().length} paths remaining.`) + const groups = _.groupBy(results.unevaluated_operations(), (operation) => operation.path.split('/')[1]) + Object.entries(groups).forEach(([root, operations]) => { + this.#log_coverage_group(root, operations) }); } - #log_coverage_group(key: string, paths: string[], index: number = 2): void { - if (paths.length == 0 || key == undefined) return - console.log(`${' '.repeat(index)}/${key} (${paths.length})`) - const current_level_paths = paths.filter((path) => path.split('/').length == index) - current_level_paths.forEach((path) => { - console.log(`${' '.repeat(index + 2)}${path}`) + #log_coverage_group(key: string, operations: Operation[], index: number = 2): void { + if (operations.length == 0 || key == undefined) return + console.log(`${' '.repeat(index)}/${key} (${operations.length})`) + const current_level_operations = operations.filter((operation) => operation.path.split('/').length == index) + current_level_operations.forEach((operation) => { + console.log(`${' '.repeat(index + 2)}${operation.method} ${operation.path}`) }) - const next_level_paths = paths.filter((path) => path.split('/').length > index) - const subgroups = _.groupBy(next_level_paths, (path) => path.split('/')[index]) - Object.entries(subgroups).forEach(([root, paths]) => { - this.#log_coverage_group(root, paths, index + 1) + const next_level_operations = operations.filter((operation) => operation.path.split('/').length > index) + const subgroups = _.groupBy(next_level_operations, (operation) => operation.path.split('/')[index]) + Object.entries(subgroups).forEach(([root, operations]) => { + this.#log_coverage_group(root, operations, index + 1) }); } diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 46371a503..b67ae5df6 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -97,8 +97,8 @@ export default class StoryEvaluator { if (multiple_paths_detected) return chapter.path })) const normalized_paths = _.map(paths, (path) => path.replaceAll(/\/\{[^}]+}/g, '').replaceAll('//', '/')) - const paths_counts: Record = Object.assign((_.values(_.groupBy(normalized_paths)).map(p => { return { [p[0]] : p.length } }))) - if (paths_counts.length > 1) { + const operations_counts: Record = Object.assign((_.values(_.groupBy(normalized_paths)).map(p => { return { [p[0]] : p.length } }))) + if (operations_counts.length > 1) { return `Multiple paths detected, please group similar tests together and move paths not being tested to prologues or epilogues.\n ${_.join(_.uniq(paths), "\n ")}\n` } } diff --git a/tools/src/tester/TestResults.ts b/tools/src/tester/TestResults.ts index 676411d20..792699e20 100644 --- a/tools/src/tester/TestResults.ts +++ b/tools/src/tester/TestResults.ts @@ -9,51 +9,59 @@ import _ from "lodash"; import MergedOpenApiSpec from "./MergedOpenApiSpec"; -import { StoryEvaluations } from "./types/eval.types"; +import { Operation, StoryEvaluations } from "./types/eval.types"; import { SpecTestCoverage } from "./types/test.types"; import { write_json } from "../helpers"; export default class TestResults { protected _spec: MergedOpenApiSpec protected _evaluations: StoryEvaluations - protected _evaluated_paths?: string[] - protected _unevaluated_paths?: string[] - protected _spec_paths?: string[] + protected _evaluated_operations?: Operation[] + protected _unevaluated_operations?: Operation[] + protected _operations?: Operation[] constructor(spec: MergedOpenApiSpec, evaluations: StoryEvaluations) { this._spec = spec this._evaluations = evaluations } - evaluated_paths(): string[] { - if (this._evaluated_paths !== undefined) return this._evaluated_paths - this._evaluated_paths = _.uniq(_.compact(_.flatten(_.map(this._evaluations.evaluations, (evaluation) => - _.map(evaluation.chapters, (chapter) => chapter.path) - )))) - return this._evaluated_paths + evaluated_operations(): Operation[] { + if (this._evaluated_operations !== undefined) return this._evaluated_operations + this._evaluated_operations = _.uniq(_.compact(_.flatMap(this._evaluations.evaluations, (evaluation) => + _.map(evaluation.chapters, (chapter) => chapter.operation) + ))) + return this._evaluated_operations } - unevaluated_paths(): string[] { - if (this._unevaluated_paths !== undefined) return this._unevaluated_paths - this._unevaluated_paths = this.spec_paths().filter((path => !this.evaluated_paths().includes(path))) - return this._unevaluated_paths + unevaluated_operations(): Operation[] { + if (this._unevaluated_operations !== undefined) return this._unevaluated_operations + this._unevaluated_operations = this.operations().filter((operation) => + !_.find(this.evaluated_operations(), + (op) => + operation.method == op.method && + operation.path == op.path + ) + ) + return this._unevaluated_operations } - spec_paths(): string[] { - if (this._spec_paths !== undefined) return this._spec_paths - this._spec_paths = _.uniq(Object.entries(this._spec.paths()).flatMap(([path, path_item]) => { - return Object.values(path_item).map((method) => `${method.toUpperCase()} ${path}`) + operations(): Operation[] { + if (this._operations !== undefined) return this._operations + this._operations = _.uniq(Object.entries(this._spec.paths()).flatMap(([path, path_item]) => { + return Object.values(path_item).map((method) => { + return { method: method.toUpperCase(), path } + }) })) - return this._spec_paths + return this._operations } test_coverage(): SpecTestCoverage { return { - evaluated_paths_count: this.evaluated_paths().length, - paths_count: this.spec_paths().length, - evaluated_paths_pct: this.spec_paths().length > 0 ? Math.round( - this.evaluated_paths().length / this.spec_paths().length * 100 * 100 + evaluated_operations_count: this.evaluated_operations().length, + total_operations_count: this.operations().length, + evaluated_paths_pct: this.operations().length > 0 ? Math.round( + this.evaluated_operations().length / this.operations().length * 100 * 100 ) / 100 : 0, } } diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts index a95104c50..eacb2c62a 100644 --- a/tools/src/tester/types/eval.types.ts +++ b/tools/src/tester/types/eval.types.ts @@ -17,6 +17,11 @@ export interface StoryFile { story: Story } +export interface Operation { + method: string + path: string +} + export interface StoryEvaluation { result: Result display_path: string @@ -36,6 +41,7 @@ export interface StoryEvaluations { export interface ChapterEvaluation { title: string, overall: Evaluation, + operation?: Operation, path?: string, request?: { parameters?: Record diff --git a/tools/src/tester/types/test.types.ts b/tools/src/tester/types/test.types.ts index c0e14717a..174f207c0 100644 --- a/tools/src/tester/types/test.types.ts +++ b/tools/src/tester/types/test.types.ts @@ -8,7 +8,7 @@ */ export interface SpecTestCoverage { - paths_count: number - evaluated_paths_count: number, + total_operations_count: number + evaluated_operations_count: number, evaluated_paths_pct: number } diff --git a/tools/tests/tester/ResultLogger.test.ts b/tools/tests/tester/ResultLogger.test.ts index 1f7e4eb53..512619cd9 100644 --- a/tools/tests/tester/ResultLogger.test.ts +++ b/tools/tests/tester/ResultLogger.test.ts @@ -66,7 +66,11 @@ describe('ConsoleResultLogger', () => { { title: 'title', overall: { result: Result.PASSED }, - path: 'path' + path: 'path', + operation: { + method: 'GET', + path: '/_nodes/{id}' + } } ] }] }) @@ -90,7 +94,11 @@ describe('ConsoleResultLogger', () => { { title: 'title', overall: { result: Result.PASSED }, - path: 'GET /_nodes/{id}' + path: 'GET /_nodes/{id}', + operation: { + method: 'GET', + path: '/_nodes/{id}' + } } ] }] }) diff --git a/tools/tests/tester/TestResults.test.ts b/tools/tests/tester/TestResults.test.ts index cf65cd1a0..d4abe8472 100644 --- a/tools/tests/tester/TestResults.test.ts +++ b/tools/tests/tester/TestResults.test.ts @@ -23,6 +23,10 @@ describe('TestResults', () => { message: 'message', chapters: [{ title: 'title', + operation: { + method: 'PUT', + path: '/{index}' + }, overall: { result: Result.PASSED }, @@ -34,29 +38,31 @@ describe('TestResults', () => { const test_results = new TestResults(spec, { evaluations }) - test('unevaluated_paths', () => { - expect(test_results.unevaluated_paths()).toEqual([ - "GET /_nodes/{id}", - "POST /_nodes/{id}", - "GET /cluster_manager", - "POST /cluster_manager", - "GET /index", - "GET /nodes" + test('unevaluated_operations', () => { + expect(test_results.unevaluated_operations()).toEqual([ + { method: "GET", path: "/_nodes/{id}" }, + { method: "POST", path: "/_nodes/{id}" }, + { method: "GET", path: "/cluster_manager" }, + { method: "POST", path: "/cluster_manager" }, + { method: "GET", path: "/index" }, + { method: "GET", path: "/nodes" } ]) }) - test('evaluated_paths', () => { - expect(test_results.evaluated_paths()).toEqual(['PUT /{index}']) + test('evaluated_operations', () => { + expect(test_results.evaluated_operations()).toStrictEqual([ + { method: 'PUT', path: '/{index}' } + ]) }) - test('spec_paths', () => { - expect(test_results.spec_paths()).toEqual([ - "GET /_nodes/{id}", - "POST /_nodes/{id}", - "GET /cluster_manager", - "POST /cluster_manager", - "GET /index", - "GET /nodes" + test('operations', () => { + expect(test_results.operations()).toEqual([ + { method: "GET", path: "/_nodes/{id}" }, + { method: "POST", path: "/_nodes/{id}" }, + { method: "GET", path: "/cluster_manager" }, + { method: "POST", path: "/cluster_manager" }, + { method: "GET", path: "/index" }, + { method: "GET", path: "/nodes" } ]) }) @@ -64,9 +70,9 @@ describe('TestResults', () => { const filename = 'coverage.json' test_results.write_coverage(filename) expect(JSON.parse(fs.readFileSync(filename, 'utf8'))).toEqual({ - evaluated_paths_count: 1, + evaluated_operations_count: 1, evaluated_paths_pct: 16.67, - paths_count: 6 + total_operations_count: 6 }) fs.unlinkSync(filename) })