Added Tests for the test framework

Signed-off-by: Theo Truong <theotr@amazon.com>

.github/opensearch-cluster/docker-compose.yml

@@ -8,4 +8,4 @@ services:
       - "9600:9600"
     environment:
       - "discovery.type=single-node"
-      - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}"
+      - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}"

DEVELOPER_GUIDE.md

@@ -145,6 +145,64 @@ This repository includes several OpenAPI Specification Extensions to fill in any
 - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml).
 - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema.
 
+## Adding tests for the spec
+
+To assure the correctness of the spec, you must add tests for the spec in the [tests/](tests) directory. Each yaml file in the tests directory represents a test story that tests a collection of related operations. A test story has 3 main components:
+- prologues: These are the operations that are executed before the test story is run. They are used to set up the environment for the test story.
+- chapters: These are the operations that are being tested.
+- epilogues: These are the operations that are executed after the test story is run. They are used to clean up the environment after the test story.
+
+Below is an example test story that tests the index operations:
+```yaml
+$schema: ../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly.
+
+skip: false # Skip this test story if set to true.
+description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion.
+
+prologues: [] # No prologues are needed for this story.
+
+epilogues: # Clean up the environment by assuring that the `books` index is deleted afterward.
+  - path: /books
+    method: DELETE
+    status: [200, 404] # The index may not exist, so we accept 404 as a valid response.
+
+chapters:
+  - synopsis: Create an index named `books` with mappings and settings.
+    path: /{index} # The test will fail if "PUT /{index}" operation is not found in the spec.
+    method: PUT
+    parameters: # All parameters are validated against their schemas in the spec
+      index: books
+    request_body: # The request body is validated against the schema of the requestBody in the spec
+      payload:
+        mappings:
+          properties:
+            name:
+              type: keyword
+            age:
+              type: integer
+        settings:
+          number_of_shards: 5
+          number_of_replicas: 2
+    response: # The response body is validated against the schema of the corresponding response in the spec
+      status: 200 # This is the expected status code of the response. Any other status code will fail the test.
+
+  - synopsis: Retrieve the mappings and settings of the `books` index.
+    path: /{index}
+    method: GET
+    parameters:
+      index: books
+      flat_settings: true
+
+  - synopsis: Delete the `books` index.
+    path: /{index}
+    method: DELETE
+    parameters:
+      index: books
+```
+
+Check the [test_story JSON Schema](json_schemas/test_story.schema.yaml) for the complete structure of a test story.
+
+
 ## Tools
 
 A number of [tools](tools) have been authored using TypeScript to aid in the development of the specification. These largely center around linting and merging the multi-file spec layout.

tools/src/tester/ChapterEvaluator.ts

@@ -21,18 +21,19 @@ export default class ChapterEvaluator {
     this.schema_validator = SharedResources.get_instance().schema_validator
   }
 
-  async evaluate (skipped: boolean): Promise<ChapterEvaluation> {
-    if (skipped) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
-    const operation = this.spec_parser.locate_operation(this.chapter)
+  async evaluate (skip: boolean): Promise<ChapterEvaluation> {
+    if (skip) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
     const response = await this.chapter_reader.read(this.chapter)
+    const operation = this.spec_parser.locate_operation(this.chapter)
+    if (operation == null) return { title: this.chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${this.chapter.method.toUpperCase()} ${this.chapter.path}" not found in the spec.` } }
     const params = this.#evaluate_parameters(operation)
     const request_body = this.#evaluate_request_body(operation)
     const status = this.#evaluate_status(response)
     const payload = this.#evaluate_payload(operation, response)
     return {
       title: this.chapter.synopsis,
       overall: { result: overall_result(Object.values(params).concat([request_body, status, payload])) },
-      request: { parameters: params, requestBody: request_body },
+      request: { parameters: params, request_body },
       response: { status, payload }
     }
   }

tools/src/tester/ResultsDisplayer.ts

@@ -48,7 +48,7 @@ export default class ResultsDisplayer {
     if (chapter.overall.result === Result.PASSED || chapter.overall.result === Result.SKIPPED) return
 
     this.#display_parameters(chapter.request?.parameters ?? {})
-    this.#display_request_body(chapter.request?.requestBody)
+    this.#display_request_body(chapter.request?.request_body)
     this.#display_status(chapter.response?.status)
     this.#display_payload(chapter.response?.payload)
   }

tools/src/tester/SchemaValidator.ts

@@ -6,7 +6,7 @@ import { type Evaluation, Result } from './types/eval.types'
 export default class SchemaValidator {
   private readonly ajv: AJV
   constructor (spec: OpenAPIV3.Document) {
-    this.ajv = new AJV()
+    this.ajv = new AJV({ allErrors: true, strict: true })
     addFormats(this.ajv)
     this.ajv.addKeyword('discriminator')
     const schemas = spec.components?.schemas ?? {}

tools/src/tester/SpecParser.ts

@@ -12,13 +12,13 @@ export default class SpecParser {
     this.spec = spec
   }
 
-  locate_operation (chapter: Chapter): ParsedOperation {
+  locate_operation (chapter: Chapter): ParsedOperation | undefined {
     const path = chapter.path
     const method = chapter.method.toLowerCase() as OpenAPIV3.HttpMethods
     const cache_key = path + method
     if (this.cached_operations[cache_key] != null) return this.cached_operations[cache_key]
     const operation = this.spec.paths[path]?.[method]
-    if (operation == null) throw new Error(`Operation "${method.toUpperCase()} ${path}" not found in the spec.`)
+    if (operation == null) return undefined
     this.#deref(operation)
     const parameters = _.keyBy(operation.parameters ?? [], 'name')
     // eslint-disable-next-line @typescript-eslint/consistent-type-assertions

tools/src/tester/StoryEvaluator.ts

@@ -50,18 +50,13 @@ export default class StoryEvaluator {
   }
 
   async #evaluate_chapters (chapters: Chapter[]): Promise<ChapterEvaluation[]> {
-    if (this.has_errors) return []
-    let has_errors: boolean = this.has_errors
-
     const evaluations: ChapterEvaluation[] = []
-
     for (const chapter of chapters) {
       const evaluator = new ChapterEvaluator(chapter)
-      const evaluation = await evaluator.evaluate(has_errors)
-      has_errors = has_errors || evaluation.overall.result === Result.ERROR
+      const evaluation = await evaluator.evaluate(this.has_errors)
+      this.has_errors = this.has_errors || evaluation.overall.result === Result.ERROR
       evaluations.push(evaluation)
     }
-
     return evaluations
   }
 
@@ -70,7 +65,7 @@ export default class StoryEvaluator {
     for (const chapter of chapters) {
       const title = `${chapter.method} ${chapter.path}`
       const response = await this.chapter_reader.read(chapter)
-      const status = chapter.status ?? []
+      const status = chapter.status ?? [200, 201]
       if (status.includes(response.status)) evaluations.push({ title, overall: { result: Result.PASSED } })
       else {
         this.has_errors = true

tools/src/tester/TestsRunner.ts

@@ -6,7 +6,7 @@ import StoryEvaluator, { type StoryFile } from './StoryEvaluator'
 import fs from 'fs'
 import { type Story } from './types/story.types'
 import { read_yaml } from '../../helpers'
-import { Result } from './types/eval.types'
+import { Result, type StoryEvaluation } from './types/eval.types'
 import ResultsDisplayer, { type DisplayOptions } from './ResultsDisplayer'
 import SharedResources from './SharedResources'
 import { resolve, basename } from 'path'
@@ -27,17 +27,20 @@ export default class TestsRunner {
     SharedResources.create_instance({ chapter_reader, schema_validator, spec_parser })
   }
 
-  async run (): Promise<void> {
+  async run (debug: boolean = false): Promise<StoryEvaluation[]> {
     let failed = false
-    const story_files = this.#collect_story_files(this.path, '', '').sort((a, b) => a.display_path.localeCompare(b.display_path))
-    for (const story_file of story_files) {
+    const story_files = this.#collect_story_files(this.path, '', '')
+    const evaluations: StoryEvaluation[] = []
+    for (const story_file of this.#sort_story_files(story_files)) {
       const evaluator = new StoryEvaluator(story_file)
       const evaluation = await evaluator.evaluate()
       const displayer = new ResultsDisplayer(evaluation, this.opts)
-      displayer.display()
+      if (debug) evaluations.push(evaluation)
+      else displayer.display()
       if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true
     }
-    if (failed) process.exit(1)
+    if (failed && !debug) process.exit(1)
+    return evaluations
   }
 
   #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] {
@@ -56,4 +59,13 @@ export default class TestsRunner {
       })
     }
   }
+
+  #sort_story_files (story_files: StoryFile[]): StoryFile[] {
+    return story_files.sort((a, b) => {
+      const a_depth = a.display_path.split('/').length
+      const b_depth = b.display_path.split('/').length
+      if (a_depth !== b_depth) return a_depth - b_depth
+      return a.display_path.localeCompare(b.display_path)
+    })
+  }
 }

tools/src/tester/types/eval.types.ts

@@ -16,7 +16,7 @@ export interface ChapterEvaluation {
   overall: Evaluation
   request?: {
     parameters?: Record<string, Evaluation>
-    requestBody?: Evaluation
+    request_body?: Evaluation
   }
   response?: {
     status: Evaluation

tools/tests/tester/StoryEvaluator.test.ts

@@ -0,0 +1,42 @@
+import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers'
+import { read_yaml } from '../../helpers'
+import { type OpenAPIV3 } from 'openapi-types'
+
+const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
+create_shared_resources(spec as OpenAPIV3.Document)
+
+test('passed', async () => {
+  const actual = await load_actual_evaluation('passed')
+  const expected = await load_expected_evaluation('passed')
+  expect(actual).toEqual(expected)
+})
+
+test('skipped', async () => {
+  const actual = await load_actual_evaluation('skipped')
+  const expected = await load_expected_evaluation('skipped')
+  expect(actual).toEqual(expected)
+})
+
+test('failed/not_found', async () => {
+  const actual = await load_actual_evaluation('failed/not_found')
+  const expected = await load_expected_evaluation('failed/not_found')
+  expect(actual).toEqual(expected)
+})
+
+test('failed/invalid_data', async () => {
+  const actual = await load_actual_evaluation('failed/invalid_data')
+  const expected = await load_expected_evaluation('failed/invalid_data')
+  expect(actual).toEqual(expected)
+})
+
+test('error/prologue_error', async () => {
+  const actual = await load_actual_evaluation('error/prologue_error')
+  const expected = await load_expected_evaluation('error/prologue_error')
+  expect(actual).toEqual(expected)
+})
+
+test('error/chapter_error', async () => {
+  const actual = await load_actual_evaluation('error/chapter_error')
+  const expected = await load_expected_evaluation('error/chapter_error')
+  expect(actual).toEqual(expected)
+})

tools/tests/tester/TestsRunner.test.ts

@@ -0,0 +1,27 @@
+import { read_yaml } from '../../helpers'
+import TestsRunner from '../../src/tester/TestsRunner'
+import { type OpenAPIV3 } from 'openapi-types'
+import { load_expected_evaluation, scrub_errors } from './helpers'
+
+test('stories folder', async () => {
+  // The password must match the one specified in .github/workflows/test-spec.yml
+  process.env.OPENSEARCH_PASSWORD = 'myStrongPassword123!'
+  const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
+  const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {})
+  const actual_evaluations = await runner.run(true) as any[]
+  for (const evaluation of actual_evaluations) scrub_errors(evaluation)
+  for (const evaluation of actual_evaluations) {
+    expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy()
+    delete evaluation.full_path
+  }
+
+  const skipped = await load_expected_evaluation('skipped', true)
+  const passed = await load_expected_evaluation('passed', true)
+  const not_found = await load_expected_evaluation('failed/not_found', true)
+  const invalid_data = await load_expected_evaluation('failed/invalid_data', true)
+  const chapter_error = await load_expected_evaluation('error/chapter_error', true)
+  const prologue_error = await load_expected_evaluation('error/prologue_error', true)
+
+  const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found]
+  expect(actual_evaluations).toEqual(expected_evaluations)
+})

tools/tests/tester/ansi.test.ts

@@ -1,17 +1,40 @@
 import * as ansi from '../../src/tester/Ansi'
 
-test('b', async () => {
+test('b', () => {
   expect(ansi.b('xyz')).toEqual('\x1b[1mxyz\x1b[0m')
 })
 
-test('i', async () => {
+test('i', () => {
   expect(ansi.i('xyz')).toEqual('\x1b[3mxyz\x1b[0m')
 })
 
-test.todo('padding')
-test.todo('green')
-test.todo('red')
-test.todo('yellow')
-test.todo('cyan')
-test.todo('gray')
-test.todo('magenta')
+test('padding', () => {
+  expect(ansi.padding('xyz', 10)).toEqual('xyz       ')
+  expect(ansi.padding('xyz', 10, 2)).toEqual('  xyz       ')
+  expect(ansi.padding('xyz', 10, 8)).toEqual('        xyz       ')
+  expect(ansi.padding('xyz', 2)).toEqual('xyz')
+})
+
+test('green', () => {
+  expect(ansi.green('xyz')).toEqual('\x1b[32mxyz\x1b[0m')
+})
+
+test('red', () => {
+  expect(ansi.red('xyz')).toEqual('\x1b[31mxyz\x1b[0m')
+})
+
+test('yellow', () => {
+  expect(ansi.yellow('xyz')).toEqual('\x1b[33mxyz\x1b[0m')
+})
+
+test('cyan', () => {
+  expect(ansi.cyan('xyz')).toEqual('\x1b[36mxyz\x1b[0m')
+})
+
+test('gray', () => {
+  expect(ansi.gray('xyz')).toEqual('\x1b[90mxyz\x1b[0m')
+})
+
+test('magenta', () => {
+  expect(ansi.magenta('xyz')).toEqual('\x1b[35mxyz\x1b[0m')
+})

tools/tests/tester/fixtures/evals/error/chapter_error.yaml

@@ -0,0 +1,39 @@
+display_path: error/chapter_error.yaml
+full_path: tools/tests/tester/fixtures/stories/error/chapter_error.yaml
+
+result: ERROR
+description: This story should failed due to missing info in the spec.
+
+prologues:
+  - title: PUT /books
+    overall:
+      result: PASSED
+
+chapters:
+  - title: This chapter should fail.
+    overall:
+      result: FAILED
+      message: Operation "GET /{index}/settings" not found in the spec.
+  - title: This chapter show throw an error.
+    overall:
+      result: ERROR
+    request:
+      parameters: {}
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: ERROR
+        message: 'Expected status 200, but received 404: application/json. no such index
+            [undefined]'
+        error: Request failed with status code 404
+      payload:
+        result: SKIPPED
+  - title: This chapter should be skipped.
+    overall:
+      result: SKIPPED
+
+epilogues:
+  - title: DELETE /books
+    overall:
+      result: PASSED