#

Signed-off-by: Theo Truong <theotr@amazon.com>
opensearch-project · May 29, 2024 · c22e52a · c22e52a
1 parent fa6949d
commit c22e52a
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 16 deletions.
diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml
@@ -55,9 +55,10 @@ definitions:
       - $ref: '#/definitions/ChapterRequest'
       - type: object
         properties:
-          additional_success_statuses:
-            description: Additional HTTP status codes, other than 2XX codes, that are considered successful.
+          status:
+            description: Array of success HTTP status codes. Default to [200, 201].
             type: array
+            default: [200, 201]
             items:
               type: integer
         additionalProperties: false

diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts
@@ -3,6 +3,7 @@ import { type ChapterEvaluation, Result, type StoryEvaluation } from './types/ev
 import ChapterEvaluator from './ChapterEvaluator'
 import type ChapterReader from './ChapterReader'
 import SharedResources from './SharedResources'
+import { overall_result } from './helpers'
 
 export interface StoryFile {
   display_path: string
@@ -15,7 +16,6 @@ export default class StoryEvaluator {
   display_path: string
   full_path: string
   has_errors: boolean = false
-  result: Result = Result.PASSED
   chapter_reader: ChapterReader
 
   constructor (story_file: StoryFile) {
@@ -35,14 +35,17 @@ export default class StoryEvaluator {
         chapters: []
       }
     }
+    const prologues = await this.#evaluate_supplemental_chapters(this.story.prologues ?? [])
+    const chapters = await this.#evaluate_chapters(this.story.chapters)
+    const epilogues = await this.#evaluate_supplemental_chapters(this.story.epilogues ?? [])
     return {
       display_path: this.display_path,
       full_path: this.full_path,
       description: this.story.description,
-      prologues: await this.#evaluate_supplemental_chapters(this.story.prologues ?? []),
-      chapters: await this.#evaluate_chapters(this.story.chapters),
-      epilogues: await this.#evaluate_supplemental_chapters(this.story.epilogues ?? []),
-      result: this.result
+      chapters,
+      prologues,
+      epilogues,
+      result: overall_result(prologues.concat(chapters).concat(epilogues).concat(prologues).map(e => e.overall))
     }
   }
 
@@ -56,8 +59,6 @@ export default class StoryEvaluator {
       const evaluator = new ChapterEvaluator(chapter)
       const evaluation = await evaluator.evaluate(has_errors)
       has_errors = has_errors || evaluation.overall.result === Result.ERROR
-      if (evaluation.overall.result === Result.FAILED) this.result = Result.FAILED
-      if (evaluation.overall.result === Result.ERROR) this.result = Result.ERROR
       evaluations.push(evaluation)
     }
 
@@ -69,11 +70,9 @@ export default class StoryEvaluator {
     for (const chapter of chapters) {
       const title = `${chapter.method} ${chapter.path}`
       const response = await this.chapter_reader.read(chapter)
-      const success_statuses = chapter.additional_success_statuses ?? []
-      if (!response.error || success_statuses.includes(response.status)) {
-        evaluations.push({ title, overall: { result: Result.PASSED } })
-      } else {
-        this.result = Result.ERROR
+      const status = chapter.status ?? []
+      if (status.includes(response.status)) evaluations.push({ title, overall: { result: Result.PASSED } })
+      else {
         this.has_errors = true
         evaluations.push({ title, overall: { result: Result.ERROR, message: response.message, error: response.error as Error } })
       }

diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts
@@ -13,9 +13,9 @@
  */
 export type SupplementalChapter = ChapterRequest & {
   /**
-   * Additional HTTP status codes, other than 2XX codes, that are considered successful.
+   * Array of success HTTP status codes. Default to [200, 201].
    */
-  additional_success_statuses?: number[];
+  status?: number[];
 };
 /**
  * This interface was referenced by `Story`'s JSON-Schema