test: add extra retries to flaky romans test (#217)

oaknational · Oct 11, 2024 · 6737c05 · 6737c05
1 parent eaf8559
commit 6737c05
Showing 1 changed file with 70 additions and 62 deletions.
diff --git a/apps/nextjs/tests-e2e/tests/aila-chat/full-romans.test.ts b/apps/nextjs/tests-e2e/tests/aila-chat/full-romans.test.ts
@@ -18,65 +18,73 @@ import {
 // const FIXTURE_MODE = "record" as FixtureMode;
 const FIXTURE_MODE = "replay" as FixtureMode;
 
-test(
-  "Full aila flow with Romans fixture",
-  { tag: "@common-auth" },
-  async ({ page }) => {
-    const generationTimeout = FIXTURE_MODE === "record" ? 75000 : 50000;
-    test.setTimeout(generationTimeout * 5);
-
-    await test.step("Setup", async () => {
-      await bypassVercelProtection(page);
-      await setupClerkTestingToken({ page });
-
-      await page.goto(`${TEST_BASE_URL}/aila`);
-      await expect(page.getByTestId("chat-h1")).toBeInViewport();
-    });
-
-    const { setFixture } = await applyLlmFixtures(page, FIXTURE_MODE);
-
-    await test.step("Fill in the chat box", async () => {
-      const textbox = page.getByTestId("chat-input");
-      const sendMessage = page.getByTestId("send-message");
-      const message =
-        "Create a KS1 lesson on the end of Roman Britain. Ask a question for each quiz and cycle";
-      await textbox.fill(message);
-      await expect(textbox).toContainText(message);
-
-      // Temporary fix: The test goes quicker than a real user and submits before the demo status has loaded
-      // This means that a demo modal would be shown when submitting
-      await page.waitForTimeout(500);
-
-      setFixture("roman-britain-1");
-      await sendMessage.click();
-    });
-
-    await test.step("Iterate through the fixtures", async () => {
-      await page.waitForURL(/\/aila\/.+/);
-      await waitForGeneration(page, generationTimeout);
-      await expectSectionsComplete(page, 1);
-
-      setFixture("roman-britain-2");
-      await continueChat(page);
-      await waitForGeneration(page, generationTimeout);
-      await expectSectionsComplete(page, 3);
-
-      setFixture("roman-britain-3");
-      await continueChat(page);
-      await waitForGeneration(page, generationTimeout);
-      await expectSectionsComplete(page, 7);
-
-      setFixture("roman-britain-4");
-      await continueChat(page);
-      await waitForGeneration(page, generationTimeout);
-      await expectSectionsComplete(page, 10);
-
-      setFixture("roman-britain-5");
-      await continueChat(page);
-      await waitForGeneration(page, generationTimeout);
-      await expectSectionsComplete(page, 10);
-
-      await expectFinished(page);
-    });
-  },
-);
+test.describe(() => {
+  // NOTE(2024-10-10): This test is flaky, with the "10 of 10" check often returning 6-8 sections
+  // Remove extra retries when it becomes more stable
+  if (process.env.CI === "true") {
+    test.describe.configure({ retries: 4 });
+  }
+
+  test(
+    "Full aila flow with Romans fixture",
+    { tag: "@common-auth" },
+    async ({ page }) => {
+      const generationTimeout = FIXTURE_MODE === "record" ? 75000 : 50000;
+      test.setTimeout(generationTimeout * 5);
+
+      await test.step("Setup", async () => {
+        await bypassVercelProtection(page);
+        await setupClerkTestingToken({ page });
+
+        await page.goto(`${TEST_BASE_URL}/aila`);
+        await expect(page.getByTestId("chat-h1")).toBeInViewport();
+      });
+
+      const { setFixture } = await applyLlmFixtures(page, FIXTURE_MODE);
+
+      await test.step("Fill in the chat box", async () => {
+        const textbox = page.getByTestId("chat-input");
+        const sendMessage = page.getByTestId("send-message");
+        const message =
+          "Create a KS1 lesson on the end of Roman Britain. Ask a question for each quiz and cycle";
+        await textbox.fill(message);
+        await expect(textbox).toContainText(message);
+
+        // Temporary fix: The test goes quicker than a real user and submits before the demo status has loaded
+        // This means that a demo modal would be shown when submitting
+        await page.waitForTimeout(500);
+
+        setFixture("roman-britain-1");
+        await sendMessage.click();
+      });
+
+      await test.step("Iterate through the fixtures", async () => {
+        await page.waitForURL(/\/aila\/.+/);
+        await waitForGeneration(page, generationTimeout);
+        await expectSectionsComplete(page, 1);
+
+        setFixture("roman-britain-2");
+        await continueChat(page);
+        await waitForGeneration(page, generationTimeout);
+        await expectSectionsComplete(page, 3);
+
+        setFixture("roman-britain-3");
+        await continueChat(page);
+        await waitForGeneration(page, generationTimeout);
+        await expectSectionsComplete(page, 7);
+
+        setFixture("roman-britain-4");
+        await continueChat(page);
+        await waitForGeneration(page, generationTimeout);
+        await expectSectionsComplete(page, 10);
+
+        setFixture("roman-britain-5");
+        await continueChat(page);
+        await waitForGeneration(page, generationTimeout);
+        await expectSectionsComplete(page, 10);
+
+        await expectFinished(page);
+      });
+    },
+  );
+});