Skip to content

Commit

Permalink
Added Tests for the test framework
Browse files Browse the repository at this point in the history
Signed-off-by: Theo Truong <theotr@amazon.com>
  • Loading branch information
nhtruong committed Jun 4, 2024
1 parent f486e8b commit 5e7b3ab
Show file tree
Hide file tree
Showing 29 changed files with 789 additions and 36 deletions.
2 changes: 1 addition & 1 deletion .github/opensearch-cluster/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ services:
- "9600:9600"
environment:
- "discovery.type=single-node"
- "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}"
- "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}"
File renamed without changes.
58 changes: 58 additions & 0 deletions DEVELOPER_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,64 @@ This repository includes several OpenAPI Specification Extensions to fill in any
- `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml).
- `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema.

## Adding tests for the spec

To assure the correctness of the spec, you must add tests for the spec in the [tests/](tests) directory. Each yaml file in the tests directory represents a test story that tests a collection of related operations. A test story has 3 main components:
- prologues: These are the operations that are executed before the test story is run. They are used to set up the environment for the test story.
- chapters: These are the operations that are being tested.
- epilogues: These are the operations that are executed after the test story is run. They are used to clean up the environment after the test story.

Below is an example test story that tests the index operations:
```yaml
$schema: ../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly.
skip: false # Skip this test story if set to true.
description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion.
prologues: [] # No prologues are needed for this story.
epilogues: # Clean up the environment by assuring that the `books` index is deleted afterward.
- path: /books
method: DELETE
status: [200, 404] # The index may not exist, so we accept 404 as a valid response.

chapters:
- synopsis: Create an index named `books` with mappings and settings.
path: /{index} # The test will fail if "PUT /{index}" operation is not found in the spec.
method: PUT
parameters: # All parameters are validated against their schemas in the spec
index: books
request_body: # The request body is validated against the schema of the requestBody in the spec
payload:
mappings:
properties:
name:
type: keyword
age:
type: integer
settings:
number_of_shards: 5
number_of_replicas: 2
response: # The response body is validated against the schema of the corresponding response in the spec
status: 200 # This is the expected status code of the response. Any other status code will fail the test.

- synopsis: Retrieve the mappings and settings of the `books` index.
path: /{index}
method: GET
parameters:
index: books
flat_settings: true

- synopsis: Delete the `books` index.
path: /{index}
method: DELETE
parameters:
index: books
```
Check the [test_story JSON Schema](json_schemas/test_story.schema.yaml) for the complete structure of a test story.
## Tools
A number of [tools](tools) have been authored using TypeScript to aid in the development of the specification. These largely center around linting and merging the multi-file spec layout.
Expand Down
9 changes: 5 additions & 4 deletions tools/src/tester/ChapterEvaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,19 @@ export default class ChapterEvaluator {
this.schema_validator = SharedResources.get_instance().schema_validator
}

async evaluate (skipped: boolean): Promise<ChapterEvaluation> {
if (skipped) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
const operation = this.spec_parser.locate_operation(this.chapter)
async evaluate (skip: boolean): Promise<ChapterEvaluation> {
if (skip) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
const response = await this.chapter_reader.read(this.chapter)
const operation = this.spec_parser.locate_operation(this.chapter)
if (operation == null) return { title: this.chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${this.chapter.method.toUpperCase()} ${this.chapter.path}" not found in the spec.` } }
const params = this.#evaluate_parameters(operation)
const request_body = this.#evaluate_request_body(operation)
const status = this.#evaluate_status(response)
const payload = this.#evaluate_payload(operation, response)
return {
title: this.chapter.synopsis,
overall: { result: overall_result(Object.values(params).concat([request_body, status, payload])) },
request: { parameters: params, requestBody: request_body },
request: { parameters: params, request_body },
response: { status, payload }
}
}
Expand Down
2 changes: 1 addition & 1 deletion tools/src/tester/ResultsDisplayer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export default class ResultsDisplayer {
if (chapter.overall.result === Result.PASSED || chapter.overall.result === Result.SKIPPED) return

this.#display_parameters(chapter.request?.parameters ?? {})
this.#display_request_body(chapter.request?.requestBody)
this.#display_request_body(chapter.request?.request_body)
this.#display_status(chapter.response?.status)
this.#display_payload(chapter.response?.payload)
}
Expand Down
2 changes: 1 addition & 1 deletion tools/src/tester/SchemaValidator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { type Evaluation, Result } from './types/eval.types'
export default class SchemaValidator {
private readonly ajv: AJV
constructor (spec: OpenAPIV3.Document) {
this.ajv = new AJV()
this.ajv = new AJV({ allErrors: true, strict: true })
addFormats(this.ajv)
this.ajv.addKeyword('discriminator')
const schemas = spec.components?.schemas ?? {}
Expand Down
4 changes: 2 additions & 2 deletions tools/src/tester/SpecParser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ export default class SpecParser {
this.spec = spec
}

locate_operation (chapter: Chapter): ParsedOperation {
locate_operation (chapter: Chapter): ParsedOperation | undefined {
const path = chapter.path
const method = chapter.method.toLowerCase() as OpenAPIV3.HttpMethods
const cache_key = path + method
if (this.cached_operations[cache_key] != null) return this.cached_operations[cache_key]
const operation = this.spec.paths[path]?.[method]
if (operation == null) throw new Error(`Operation "${method.toUpperCase()} ${path}" not found in the spec.`)
if (operation == null) return undefined
this.#deref(operation)
const parameters = _.keyBy(operation.parameters ?? [], 'name')
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
Expand Down
11 changes: 3 additions & 8 deletions tools/src/tester/StoryEvaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,18 +50,13 @@ export default class StoryEvaluator {
}

async #evaluate_chapters (chapters: Chapter[]): Promise<ChapterEvaluation[]> {
if (this.has_errors) return []
let has_errors: boolean = this.has_errors

const evaluations: ChapterEvaluation[] = []

for (const chapter of chapters) {
const evaluator = new ChapterEvaluator(chapter)
const evaluation = await evaluator.evaluate(has_errors)
has_errors = has_errors || evaluation.overall.result === Result.ERROR
const evaluation = await evaluator.evaluate(this.has_errors)
this.has_errors = this.has_errors || evaluation.overall.result === Result.ERROR
evaluations.push(evaluation)
}

return evaluations
}

Expand All @@ -70,7 +65,7 @@ export default class StoryEvaluator {
for (const chapter of chapters) {
const title = `${chapter.method} ${chapter.path}`
const response = await this.chapter_reader.read(chapter)
const status = chapter.status ?? []
const status = chapter.status ?? [200, 201]
if (status.includes(response.status)) evaluations.push({ title, overall: { result: Result.PASSED } })
else {
this.has_errors = true
Expand Down
24 changes: 18 additions & 6 deletions tools/src/tester/TestsRunner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import StoryEvaluator, { type StoryFile } from './StoryEvaluator'
import fs from 'fs'
import { type Story } from './types/story.types'
import { read_yaml } from '../../helpers'
import { Result } from './types/eval.types'
import { Result, type StoryEvaluation } from './types/eval.types'
import ResultsDisplayer, { type DisplayOptions } from './ResultsDisplayer'
import SharedResources from './SharedResources'
import { resolve, basename } from 'path'
Expand All @@ -27,17 +27,20 @@ export default class TestsRunner {
SharedResources.create_instance({ chapter_reader, schema_validator, spec_parser })
}

async run (): Promise<void> {
async run (debug: boolean = false): Promise<StoryEvaluation[]> {
let failed = false
const story_files = this.#collect_story_files(this.path, '', '').sort((a, b) => a.display_path.localeCompare(b.display_path))
for (const story_file of story_files) {
const story_files = this.#collect_story_files(this.path, '', '')
const evaluations: StoryEvaluation[] = []
for (const story_file of this.#sort_story_files(story_files)) {
const evaluator = new StoryEvaluator(story_file)
const evaluation = await evaluator.evaluate()
const displayer = new ResultsDisplayer(evaluation, this.opts)
displayer.display()
if (debug) evaluations.push(evaluation)
else displayer.display()
if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true
}
if (failed) process.exit(1)
if (failed && !debug) process.exit(1)
return evaluations
}

#collect_story_files (folder: string, file: string, prefix: string): StoryFile[] {
Expand All @@ -56,4 +59,13 @@ export default class TestsRunner {
})
}
}

#sort_story_files (story_files: StoryFile[]): StoryFile[] {
return story_files.sort((a, b) => {
const a_depth = a.display_path.split('/').length
const b_depth = b.display_path.split('/').length
if (a_depth !== b_depth) return a_depth - b_depth
return a.display_path.localeCompare(b.display_path)
})
}
}
2 changes: 1 addition & 1 deletion tools/src/tester/types/eval.types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export interface ChapterEvaluation {
overall: Evaluation
request?: {
parameters?: Record<string, Evaluation>
requestBody?: Evaluation
request_body?: Evaluation
}
response?: {
status: Evaluation
Expand Down
42 changes: 42 additions & 0 deletions tools/tests/tester/StoryEvaluator.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers'
import { read_yaml } from '../../helpers'
import { type OpenAPIV3 } from 'openapi-types'

const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
create_shared_resources(spec as OpenAPIV3.Document)

test('passed', async () => {
const actual = await load_actual_evaluation('passed')
const expected = await load_expected_evaluation('passed')
expect(actual).toEqual(expected)
})

test('skipped', async () => {
const actual = await load_actual_evaluation('skipped')
const expected = await load_expected_evaluation('skipped')
expect(actual).toEqual(expected)
})

test('failed/not_found', async () => {
const actual = await load_actual_evaluation('failed/not_found')
const expected = await load_expected_evaluation('failed/not_found')
expect(actual).toEqual(expected)
})

test('failed/invalid_data', async () => {
const actual = await load_actual_evaluation('failed/invalid_data')
const expected = await load_expected_evaluation('failed/invalid_data')
expect(actual).toEqual(expected)
})

test('error/prologue_error', async () => {
const actual = await load_actual_evaluation('error/prologue_error')
const expected = await load_expected_evaluation('error/prologue_error')
expect(actual).toEqual(expected)
})

test('error/chapter_error', async () => {
const actual = await load_actual_evaluation('error/chapter_error')
const expected = await load_expected_evaluation('error/chapter_error')
expect(actual).toEqual(expected)
})
27 changes: 27 additions & 0 deletions tools/tests/tester/TestsRunner.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { read_yaml } from '../../helpers'
import TestsRunner from '../../src/tester/TestsRunner'
import { type OpenAPIV3 } from 'openapi-types'
import { load_expected_evaluation, scrub_errors } from './helpers'

test('stories folder', async () => {
// The password must match the one specified in .github/workflows/test-spec.yml
process.env.OPENSEARCH_PASSWORD = 'myStrongPassword123!'
const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {})
const actual_evaluations = await runner.run(true) as any[]
for (const evaluation of actual_evaluations) scrub_errors(evaluation)
for (const evaluation of actual_evaluations) {
expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy()
delete evaluation.full_path
}

const skipped = await load_expected_evaluation('skipped', true)
const passed = await load_expected_evaluation('passed', true)
const not_found = await load_expected_evaluation('failed/not_found', true)
const invalid_data = await load_expected_evaluation('failed/invalid_data', true)
const chapter_error = await load_expected_evaluation('error/chapter_error', true)
const prologue_error = await load_expected_evaluation('error/prologue_error', true)

const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found]
expect(actual_evaluations).toEqual(expected_evaluations)
})
41 changes: 32 additions & 9 deletions tools/tests/tester/ansi.test.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,40 @@
import * as ansi from '../../src/tester/Ansi'

test('b', async () => {
test('b', () => {
expect(ansi.b('xyz')).toEqual('\x1b[1mxyz\x1b[0m')
})

test('i', async () => {
test('i', () => {
expect(ansi.i('xyz')).toEqual('\x1b[3mxyz\x1b[0m')
})

test.todo('padding')
test.todo('green')
test.todo('red')
test.todo('yellow')
test.todo('cyan')
test.todo('gray')
test.todo('magenta')
test('padding', () => {
expect(ansi.padding('xyz', 10)).toEqual('xyz ')
expect(ansi.padding('xyz', 10, 2)).toEqual(' xyz ')
expect(ansi.padding('xyz', 10, 8)).toEqual(' xyz ')
expect(ansi.padding('xyz', 2)).toEqual('xyz')
})

test('green', () => {
expect(ansi.green('xyz')).toEqual('\x1b[32mxyz\x1b[0m')
})

test('red', () => {
expect(ansi.red('xyz')).toEqual('\x1b[31mxyz\x1b[0m')
})

test('yellow', () => {
expect(ansi.yellow('xyz')).toEqual('\x1b[33mxyz\x1b[0m')
})

test('cyan', () => {
expect(ansi.cyan('xyz')).toEqual('\x1b[36mxyz\x1b[0m')
})

test('gray', () => {
expect(ansi.gray('xyz')).toEqual('\x1b[90mxyz\x1b[0m')
})

test('magenta', () => {
expect(ansi.magenta('xyz')).toEqual('\x1b[35mxyz\x1b[0m')
})
3 changes: 0 additions & 3 deletions tools/tests/tester/fixtures/empty.yaml

This file was deleted.

39 changes: 39 additions & 0 deletions tools/tests/tester/fixtures/evals/error/chapter_error.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
display_path: error/chapter_error.yaml
full_path: tools/tests/tester/fixtures/stories/error/chapter_error.yaml

result: ERROR
description: This story should failed due to missing info in the spec.

prologues:
- title: PUT /books
overall:
result: PASSED

chapters:
- title: This chapter should fail.
overall:
result: FAILED
message: Operation "GET /{index}/settings" not found in the spec.
- title: This chapter show throw an error.
overall:
result: ERROR
request:
parameters: {}
request_body:
result: PASSED
response:
status:
result: ERROR
message: 'Expected status 200, but received 404: application/json. no such index
[undefined]'
error: Request failed with status code 404
payload:
result: SKIPPED
- title: This chapter should be skipped.
overall:
result: SKIPPED

epilogues:
- title: DELETE /books
overall:
result: PASSED
Loading

0 comments on commit 5e7b3ab

Please sign in to comment.