diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 344ac9d2f..55fcb6cc1 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -55,9 +55,10 @@ definitions: - $ref: '#/definitions/ChapterRequest' - type: object properties: - additional_success_statuses: - description: Additional HTTP status codes, other than 2XX codes, that are considered successful. + status: + description: Array of success HTTP status codes. Default to [200, 201]. type: array + default: [200, 201] items: type: integer additionalProperties: false diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 395c0e0fb..7c049125c 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -3,6 +3,7 @@ import { type ChapterEvaluation, Result, type StoryEvaluation } from './types/ev import ChapterEvaluator from './ChapterEvaluator' import type ChapterReader from './ChapterReader' import SharedResources from './SharedResources' +import { overall_result } from './helpers' export interface StoryFile { display_path: string @@ -15,7 +16,6 @@ export default class StoryEvaluator { display_path: string full_path: string has_errors: boolean = false - result: Result = Result.PASSED chapter_reader: ChapterReader constructor (story_file: StoryFile) { @@ -35,14 +35,17 @@ export default class StoryEvaluator { chapters: [] } } + const prologues = await this.#evaluate_supplemental_chapters(this.story.prologues ?? []) + const chapters = await this.#evaluate_chapters(this.story.chapters) + const epilogues = await this.#evaluate_supplemental_chapters(this.story.epilogues ?? []) return { display_path: this.display_path, full_path: this.full_path, description: this.story.description, - prologues: await this.#evaluate_supplemental_chapters(this.story.prologues ?? []), - chapters: await this.#evaluate_chapters(this.story.chapters), - epilogues: await this.#evaluate_supplemental_chapters(this.story.epilogues ?? []), - result: this.result + chapters, + prologues, + epilogues, + result: overall_result(prologues.concat(chapters).concat(epilogues).concat(prologues).map(e => e.overall)) } } @@ -56,8 +59,6 @@ export default class StoryEvaluator { const evaluator = new ChapterEvaluator(chapter) const evaluation = await evaluator.evaluate(has_errors) has_errors = has_errors || evaluation.overall.result === Result.ERROR - if (evaluation.overall.result === Result.FAILED) this.result = Result.FAILED - if (evaluation.overall.result === Result.ERROR) this.result = Result.ERROR evaluations.push(evaluation) } @@ -69,11 +70,9 @@ export default class StoryEvaluator { for (const chapter of chapters) { const title = `${chapter.method} ${chapter.path}` const response = await this.chapter_reader.read(chapter) - const success_statuses = chapter.additional_success_statuses ?? [] - if (!response.error || success_statuses.includes(response.status)) { - evaluations.push({ title, overall: { result: Result.PASSED } }) - } else { - this.result = Result.ERROR + const status = chapter.status ?? [] + if (status.includes(response.status)) evaluations.push({ title, overall: { result: Result.PASSED } }) + else { this.has_errors = true evaluations.push({ title, overall: { result: Result.ERROR, message: response.message, error: response.error as Error } }) } diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 67e2b0288..344a15880 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -13,9 +13,9 @@ */ export type SupplementalChapter = ChapterRequest & { /** - * Additional HTTP status codes, other than 2XX codes, that are considered successful. + * Array of success HTTP status codes. Default to [200, 201]. */ - additional_success_statuses?: number[]; + status?: number[]; }; /** * This interface was referenced by `Story`'s JSON-Schema