Refactored paths into operations.

Signed-off-by: dblock <dblock@amazon.com>

.github/pr-comment-templates/pr-test-coverage-analysis.template.md

@@ -1,8 +1,8 @@
 ## Spec Test Coverage Analysis
 {{with .test_coverage}}
 
-| Total             | Tested                                                   |
-|-------------------|----------------------------------------------------------|
-| {{.paths_count}}  | {{.evaluated_paths_count}} ({{.evaluated_paths_pct}} %)  |
+| Total                        | Tested                                                        |
+|------------------------------|---------------------------------------------------------------|
+| {{.total_operations_count}}  | {{.evaluated_operations_count}} ({{.evaluated_paths_pct}} %)  |
 
 {{end}}

TESTING_GUIDE.md

@@ -330,8 +330,8 @@ The test tool can generate a test coverage summary using `--coverage <path>` wit
 
 ```json
 {
-  "evaluated_paths_count": 214,
-  "paths_count": 550,
+  "evaluated_operations_count": 214,
+  "operations_count": 550,
   "evaluated_paths_pct": 38.91
 }
 ```

tools/src/tester/ChapterEvaluator.ts

@@ -84,6 +84,10 @@ export default class ChapterEvaluator {
 
     var result: ChapterEvaluation = {
       title: chapter.synopsis,
+      operation: {
+        method: chapter.method,
+        path: chapter.path
+      },
       path: `${chapter.method} ${chapter.path}`,
       overall: { result: overall_result(evaluations) },
       request: { parameters: params, request },

tools/src/tester/ResultLogger.ts

@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { type ChapterEvaluation, type Evaluation, Result, type StoryEvaluation } from './types/eval.types'
+import { type ChapterEvaluation, type Evaluation, Operation, Result, type StoryEvaluation } from './types/eval.types'
 import { overall_result } from './helpers'
 import * as ansi from './Ansi'
 import TestResults from './TestResults'
@@ -44,29 +44,29 @@ export class ConsoleResultLogger implements ResultLogger {
 
   log_coverage(results: TestResults): void {
     console.log()
-    console.log(`Tested ${results.evaluated_paths().length}/${results.spec_paths().length} paths.`)
+    console.log(`Tested ${results.evaluated_operations().length}/${results.operations().length} paths.`)
   }
 
   log_coverage_report(results: TestResults): void {
     console.log()
-    console.log(`${results.unevaluated_paths().length} paths remaining.`)
-    const groups = _.groupBy(results.unevaluated_paths(), (path) => path.split(' ', 2)[1].split('/')[1])
-    Object.entries(groups).forEach(([root, paths]) => {
-      this.#log_coverage_group(root, paths)
+    console.log(`${results.unevaluated_operations().length} paths remaining.`)
+    const groups = _.groupBy(results.unevaluated_operations(), (operation) => operation.path.split('/')[1])
+    Object.entries(groups).forEach(([root, operations]) => {
+      this.#log_coverage_group(root, operations)
     });
   }
 
-  #log_coverage_group(key: string, paths: string[], index: number = 2): void {
-    if (paths.length == 0 || key == undefined) return
-    console.log(`${' '.repeat(index)}/${key} (${paths.length})`)
-    const current_level_paths = paths.filter((path) => path.split('/').length == index)
-    current_level_paths.forEach((path) => {
-      console.log(`${' '.repeat(index + 2)}${path}`)
+  #log_coverage_group(key: string, operations: Operation[], index: number = 2): void {
+    if (operations.length == 0 || key == undefined) return
+    console.log(`${' '.repeat(index)}/${key} (${operations.length})`)
+    const current_level_operations = operations.filter((operation) => operation.path.split('/').length == index)
+    current_level_operations.forEach((operation) => {
+      console.log(`${' '.repeat(index + 2)}${operation.method} ${operation.path}`)
     })
-    const next_level_paths = paths.filter((path) => path.split('/').length > index)
-    const subgroups = _.groupBy(next_level_paths, (path) => path.split('/')[index])
-    Object.entries(subgroups).forEach(([root, paths]) => {
-      this.#log_coverage_group(root, paths, index + 1)
+    const next_level_operations = operations.filter((operation) => operation.path.split('/').length > index)
+    const subgroups = _.groupBy(next_level_operations, (operation) => operation.path.split('/')[index])
+    Object.entries(subgroups).forEach(([root, operations]) => {
+      this.#log_coverage_group(root, operations, index + 1)
     });
   }
 

tools/src/tester/StoryEvaluator.ts

@@ -97,8 +97,8 @@ export default class StoryEvaluator {
       if (multiple_paths_detected) return chapter.path
     }))
     const normalized_paths = _.map(paths, (path) => path.replaceAll(/\/\{[^}]+}/g, '').replaceAll('//', '/'))
-    const paths_counts: Record<string, number> = Object.assign((_.values(_.groupBy(normalized_paths)).map(p => { return { [p[0]] : p.length } })))
-    if (paths_counts.length > 1) {
+    const operations_counts: Record<string, number> = Object.assign((_.values(_.groupBy(normalized_paths)).map(p => { return { [p[0]] : p.length } })))
+    if (operations_counts.length > 1) {
       return `Multiple paths detected, please group similar tests together and move paths not being tested to prologues or epilogues.\n  ${_.join(_.uniq(paths), "\n  ")}\n`
     }
   }

tools/src/tester/TestResults.ts

@@ -9,51 +9,59 @@
 
 import _ from "lodash";
 import MergedOpenApiSpec from "./MergedOpenApiSpec";
-import { StoryEvaluations } from "./types/eval.types";
+import { Operation, StoryEvaluations } from "./types/eval.types";
 import { SpecTestCoverage } from "./types/test.types";
 import { write_json } from "../helpers";
 
 export default class TestResults {
   protected _spec: MergedOpenApiSpec
   protected _evaluations: StoryEvaluations
-  protected _evaluated_paths?: string[]
-  protected _unevaluated_paths?: string[]
-  protected _spec_paths?: string[]
+  protected _evaluated_operations?: Operation[]
+  protected _unevaluated_operations?: Operation[]
+  protected _operations?: Operation[]
 
   constructor(spec: MergedOpenApiSpec, evaluations: StoryEvaluations) {
     this._spec = spec
     this._evaluations = evaluations
   }
 
-  evaluated_paths(): string[] {
-    if (this._evaluated_paths !== undefined) return this._evaluated_paths
-    this._evaluated_paths = _.uniq(_.compact(_.flatten(_.map(this._evaluations.evaluations, (evaluation) =>
-      _.map(evaluation.chapters, (chapter) => chapter.path)
-    ))))
-    return this._evaluated_paths
+  evaluated_operations(): Operation[] {
+    if (this._evaluated_operations !== undefined) return this._evaluated_operations
+    this._evaluated_operations = _.uniq(_.compact(_.flatMap(this._evaluations.evaluations, (evaluation) =>
+      _.map(evaluation.chapters, (chapter) => chapter.operation)
+    )))
+    return this._evaluated_operations
   }
 
-  unevaluated_paths(): string[] {
-    if (this._unevaluated_paths !== undefined) return this._unevaluated_paths
-    this._unevaluated_paths = this.spec_paths().filter((path => !this.evaluated_paths().includes(path)))
-    return this._unevaluated_paths
+  unevaluated_operations(): Operation[] {
+    if (this._unevaluated_operations !== undefined) return this._unevaluated_operations
+    this._unevaluated_operations = this.operations().filter((operation) =>
+      !_.find(this.evaluated_operations(),
+        (op) =>
+          operation.method == op.method &&
+          operation.path == op.path
+      )
+    )
+    return this._unevaluated_operations
   }
 
-  spec_paths(): string[] {
-    if (this._spec_paths !== undefined) return this._spec_paths
-    this._spec_paths = _.uniq(Object.entries(this._spec.paths()).flatMap(([path, path_item]) => {
-      return Object.values(path_item).map((method) => `${method.toUpperCase()} ${path}`)
+  operations(): Operation[] {
+    if (this._operations !== undefined) return this._operations
+    this._operations = _.uniq(Object.entries(this._spec.paths()).flatMap(([path, path_item]) => {
+      return Object.values(path_item).map((method) => {
+        return { method: method.toUpperCase(), path }
+      })
     }))
 
-    return this._spec_paths
+    return this._operations
   }
 
   test_coverage(): SpecTestCoverage {
     return {
-      evaluated_paths_count: this.evaluated_paths().length,
-      paths_count: this.spec_paths().length,
-      evaluated_paths_pct: this.spec_paths().length > 0 ? Math.round(
-        this.evaluated_paths().length / this.spec_paths().length * 100 * 100
+      evaluated_operations_count: this.evaluated_operations().length,
+      total_operations_count: this.operations().length,
+      evaluated_paths_pct: this.operations().length > 0 ? Math.round(
+        this.evaluated_operations().length / this.operations().length * 100 * 100
       ) / 100 : 0,
     }
   }

tools/src/tester/types/eval.types.ts

@@ -17,6 +17,11 @@ export interface StoryFile {
   story: Story
 }
 
+export interface Operation {
+  method: string
+  path: string
+}
+
 export interface StoryEvaluation {
   result: Result
   display_path: string
@@ -36,6 +41,7 @@ export interface StoryEvaluations {
 export interface ChapterEvaluation {
   title: string,
   overall: Evaluation,
+  operation?: Operation,
   path?: string,
   request?: {
     parameters?: Record<string, Evaluation>

tools/src/tester/types/test.types.ts

@@ -8,7 +8,7 @@
 */
 
 export interface SpecTestCoverage {
-  paths_count: number
-  evaluated_paths_count: number,
+  total_operations_count: number
+  evaluated_operations_count: number,
   evaluated_paths_pct: number
 }

tools/tests/tester/ResultLogger.test.ts

@@ -66,7 +66,11 @@ describe('ConsoleResultLogger', () => {
           {
             title: 'title',
             overall: { result: Result.PASSED },
-            path: 'path'
+            path: 'path',
+            operation: {
+              method: 'GET',
+              path: '/_nodes/{id}'
+            }
           }
         ]
       }] })
@@ -90,7 +94,11 @@ describe('ConsoleResultLogger', () => {
           {
             title: 'title',
             overall: { result: Result.PASSED },
-            path: 'GET /_nodes/{id}'
+            path: 'GET /_nodes/{id}',
+            operation: {
+              method: 'GET',
+              path: '/_nodes/{id}'
+            }
           }
         ]
       }] })

tools/tests/tester/TestResults.test.ts

@@ -23,6 +23,10 @@ describe('TestResults', () => {
     message: 'message',
     chapters: [{
       title: 'title',
+      operation: {
+        method: 'PUT',
+        path: '/{index}'
+      },
       overall: {
         result: Result.PASSED
       },
@@ -34,39 +38,41 @@ describe('TestResults', () => {
 
   const test_results = new TestResults(spec, { evaluations })
 
-  test('unevaluated_paths', () => {
-    expect(test_results.unevaluated_paths()).toEqual([
-      "GET /_nodes/{id}",
-      "POST /_nodes/{id}",
-      "GET /cluster_manager",
-      "POST /cluster_manager",
-      "GET /index",
-      "GET /nodes"
+  test('unevaluated_operations', () => {
+    expect(test_results.unevaluated_operations()).toEqual([
+      { method: "GET", path: "/_nodes/{id}" },
+      { method: "POST", path: "/_nodes/{id}" },
+      { method: "GET", path: "/cluster_manager" },
+      { method: "POST", path: "/cluster_manager" },
+      { method: "GET", path: "/index" },
+      { method: "GET", path: "/nodes" }
     ])
   })
 
-  test('evaluated_paths', () => {
-    expect(test_results.evaluated_paths()).toEqual(['PUT /{index}'])
+  test('evaluated_operations', () => {
+    expect(test_results.evaluated_operations()).toStrictEqual([
+      { method: 'PUT', path: '/{index}' }
+    ])
   })
 
-  test('spec_paths', () => {
-    expect(test_results.spec_paths()).toEqual([
-      "GET /_nodes/{id}",
-      "POST /_nodes/{id}",
-      "GET /cluster_manager",
-      "POST /cluster_manager",
-      "GET /index",
-      "GET /nodes"
+  test('operations', () => {
+    expect(test_results.operations()).toEqual([
+      { method: "GET", path: "/_nodes/{id}" },
+      { method: "POST", path: "/_nodes/{id}" },
+      { method: "GET", path: "/cluster_manager" },
+      { method: "POST", path: "/cluster_manager" },
+      { method: "GET", path: "/index" },
+      { method: "GET", path: "/nodes" }
     ])
   })
 
   test('write_coverage', () => {
     const filename = 'coverage.json'
     test_results.write_coverage(filename)
     expect(JSON.parse(fs.readFileSync(filename, 'utf8'))).toEqual({
-      evaluated_paths_count: 1,
+      evaluated_operations_count: 1,
       evaluated_paths_pct: 16.67,
-      paths_count: 6
+      total_operations_count: 6
     })
     fs.unlinkSync(filename)
   })