Remove cloning and user-reset from e2e tests (#10418)

Resolves FE-2347 and BAC-4754. This was originally added in #9497 as a way to avoid parallel tests from interfering with each other but has since proved to be difficult to maintain on the backend side. Our tests will now use unique "scopes" instead to isolate comments, points, and nags between runs. I've tested this locally by... - Running tests in parallel, multiple times - Killing tests in the middle (leaving stranded comments) and then re-running them
replayio · Mar 8, 2024 · f45d517 · f45d517
1 parent 5cd4aff
commit f45d517
Show file tree

Hide file tree

Showing 107 changed files with 112 additions and 192 deletions.
diff --git a/packages/e2e-tests/README.md b/packages/e2e-tests/README.md
@@ -56,7 +56,7 @@ Our old E2E test suite re-recorded those examples every time the test suite ran.
 The tests themselves are written using a set of "page object"-style helper functions that abstract specific pieces of DOM interaction. This allows the tests themselves to be written in a more readable style and focus on the sequence of operations:
 
 ```ts
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test(`Test stepping forward through breakpoints when rewound before the first one.`, async ({
   pageWithMeta: { page, recordingId },

diff --git a/packages/e2e-tests/authenticated/comments-01.test.ts b/packages/e2e-tests/authenticated/comments-01.test.ts
@@ -2,7 +2,7 @@ import { openDevToolsTab, startTest } from "../helpers";
 import { E2E_USER_1 } from "../helpers/authentication";
 import { addSourceCodeComment, deleteComment, editComment } from "../helpers/comments";
 import { openSource } from "../helpers/source-explorer-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "authenticated_comments.html", testUsers: [E2E_USER_1] });
 

diff --git a/packages/e2e-tests/authenticated/comments-02.test.ts b/packages/e2e-tests/authenticated/comments-02.test.ts
@@ -7,7 +7,7 @@ import {
   replyToComment,
 } from "../helpers/comments";
 import { openSource } from "../helpers/source-explorer-panel";
-import test, { Page } from "../testFixtureCloneRecording";
+import test, { Page } from "../testFixture";
 
 const url = "authenticated_comments.html";
 

diff --git a/packages/e2e-tests/authenticated/comments-03.test.ts b/packages/e2e-tests/authenticated/comments-03.test.ts
@@ -9,7 +9,7 @@ import {
 } from "../helpers/comments";
 import { openNetworkPanel } from "../helpers/network-panel";
 import { openSource } from "../helpers/source-explorer-panel";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "authenticated_comments.html", testUsers: [E2E_USER_1] });
 

diff --git a/packages/e2e-tests/authenticated/logpoints-01.test.ts b/packages/e2e-tests/authenticated/logpoints-01.test.ts
@@ -10,7 +10,7 @@ import {
 import { openSource } from "../helpers/source-explorer-panel";
 import { addLogpoint, editLogPoint, removeAllLogpoints } from "../helpers/source-panel";
 import { waitForRecordingToFinishIndexing } from "../helpers/utils";
-import test, { Page, expect } from "../testFixtureCloneRecording";
+import test, { Page, expect } from "../testFixture";
 
 const url = "authenticated_logpoints.html";
 const lineNumber = 14;

diff --git a/packages/e2e-tests/authenticated/passport-01.test.ts b/packages/e2e-tests/authenticated/passport-01.test.ts
@@ -5,7 +5,7 @@ import { isPassportItemCompleted, showPassport } from "../helpers/passport";
 import { enablePassport } from "../helpers/settings";
 import { addLogpoint, removeAllLogpoints } from "../helpers/source-panel";
 import { waitFor } from "../helpers/utils";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_console.html", testUsers: [E2E_USER_1] });
 

diff --git a/packages/e2e-tests/authenticated/passport-02.test.ts b/packages/e2e-tests/authenticated/passport-02.test.ts
@@ -14,7 +14,7 @@ import { openReactDevtoolsPanel } from "../helpers/new-react-devtools-panel";
 import { isPassportItemCompleted } from "../helpers/passport";
 import { enablePassport } from "../helpers/settings";
 import { waitFor } from "../helpers/utils";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "cra/dist/index.html", testUsers: [E2E_USER_1] });
 

diff --git a/packages/e2e-tests/authenticated/passport-03.test.ts b/packages/e2e-tests/authenticated/passport-03.test.ts
@@ -5,7 +5,7 @@ import { enablePassport } from "../helpers/settings";
 import { addLogpoint, editLogPoint, removeAllLogpoints } from "../helpers/source-panel";
 import { enterFocusMode, saveFocusRange } from "../helpers/timeline";
 import { clearTextArea, getCommandKey, waitFor } from "../helpers/utils";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_console.html", testUsers: [E2E_USER_1] });
 

diff --git a/packages/e2e-tests/authenticated/passport-04.test.ts b/packages/e2e-tests/authenticated/passport-04.test.ts
@@ -4,7 +4,7 @@ import { addSourceCodeComment, deleteComment } from "../helpers/comments";
 import { isPassportItemCompleted } from "../helpers/passport";
 import { enablePassport } from "../helpers/settings";
 import { waitFor } from "../helpers/utils";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_console.html", testUsers: [E2E_USER_1] });
 

diff --git a/packages/e2e-tests/helpers/utils.ts b/packages/e2e-tests/helpers/utils.ts
@@ -268,67 +268,3 @@ export async function resetTestUser(email: string, testScope: string) {
     },
   });
 }
-
-export async function cloneTestRecording(recordingId: string): Promise<string> {
-  if (!process.env.AUTHENTICATED_TESTS_WORKSPACE_API_KEY) {
-    throw new Error(
-      "AUTHENTICATED_TESTS_WORKSPACE_API_KEY must be set in order to clone test recordings."
-    );
-  }
-
-  const variables = { recordingId, secret: process.env.AUTOMATED_TEST_SECRET };
-
-  var startTime = performance.now();
-  const clonedRecording = await axios({
-    url: config.graphqlUrl,
-    method: "POST",
-    headers: {
-      Authorization: `Bearer ${process.env.AUTHENTICATED_TESTS_WORKSPACE_API_KEY}`,
-    },
-    data: {
-      query: `
-        mutation cloneTestRecording($recordingId: UUID!, $secret: String!) {
-          cloneTestRecording(input: { recordingId: $recordingId, secret: $secret }) {
-            recordingId
-          }
-        }
-      `,
-      variables,
-    },
-  });
-  var endTime = performance.now();
-  const timeInSecs = Math.round((endTime - startTime) / 1000);
-  if (timeInSecs > 10) {
-    console.warn(`Cloning took ${timeInSecs} seconds.`);
-  }
-
-  return clonedRecording.data.data.cloneTestRecording.recordingId;
-}
-
-export async function deleteTestRecording(recordingId: string) {
-  if (!process.env.AUTHENTICATED_TESTS_WORKSPACE_API_KEY) {
-    throw new Error(
-      "AUTHENTICATED_TESTS_WORKSPACE_API_KEY must be set in order to delete test recordings."
-    );
-  }
-
-  const variables = { recordingId, secret: process.env.AUTOMATED_TEST_SECRET };
-
-  return axios({
-    url: config.graphqlUrl,
-    method: "POST",
-    headers: {
-      Authorization: `Bearer ${process.env.AUTHENTICATED_TESTS_WORKSPACE_API_KEY}`,
-    },
-    data: {
-      query: `
-          mutation deleteTestRecording($recordingId: UUID!, $secret: String!) {
-            deleteTestRecording(input: { recordingId: $recordingId, secret: $secret }) {
-              success
-            }
-          }
-        `,
-      variables,
-    },
-  });
-}
diff --git a/...es/e2e-tests/testFixtureCloneRecording.ts → packages/e2e-tests/testFixture.ts b/...es/e2e-tests/testFixtureCloneRecording.ts → packages/e2e-tests/testFixture.ts
@@ -1,10 +1,10 @@
 import crypto from "crypto";
 import { Page, test as base } from "@playwright/test";
-import axios from "axios";
 
+import exampleRecordings from "./examples.json";
 import { TestRecordingKey } from "./helpers";
 import { TestUser } from "./helpers/authentication";
-import { cloneTestRecording, deleteTestRecording, resetTestUser } from "./helpers/utils";
+import { resetTestUser } from "./helpers/utils";
 import { loadRecording } from "./scripts/loadRecording";
 
 type TestIsolatedRecordingFixture = {
@@ -23,30 +23,16 @@ const testWithCloneRecording = base.extend<TestIsolatedRecordingFixture>({
   exampleKey: undefined,
   testUsers: undefined,
   pageWithMeta: async ({ page, exampleKey, testUsers }, use) => {
-    const exampleRecordings = require("./examples.json");
     if (!exampleRecordings[exampleKey]) {
       throw new Error(`Invalid recording: ${exampleKey}`);
     }
 
     const testScope = crypto.randomUUID();
-    let newRecordingId: string | undefined = undefined;
-    try {
-      try {
-        const { recording } = exampleRecordings[exampleKey];
-        console.log("Cloning recording");
-        newRecordingId = await cloneTestRecording(recording);
-      } catch (err: any) {
-        if (axios.isAxiosError(err)) {
-          console.error("Axios error cloning recording: ", {
-            errors: err.response?.data?.errors,
-            details: err.toJSON(),
-          });
-        }
-        throw err;
-      }
 
+    let recordingId = exampleRecordings[exampleKey]?.recording;
+    try {
       try {
-        await loadRecording(newRecordingId);
+        await loadRecording(recordingId);
       } catch (e) {
         console.warn("Error processing recording; ignoring.");
       }
@@ -59,7 +45,7 @@ const testWithCloneRecording = base.extend<TestIsolatedRecordingFixture>({
       // Run test.
       await use({
         page,
-        recordingId: newRecordingId,
+        recordingId: recordingId,
         testScope,
       });
     } finally {
@@ -69,15 +55,13 @@ const testWithCloneRecording = base.extend<TestIsolatedRecordingFixture>({
         console.error("Error stopping JS coverage: ", err);
       }
 
-      if (newRecordingId) {
-        await deleteTestRecording(newRecordingId);
-      }
       for (const user of testUsers ?? []) {
         await resetTestUser(user.email, testScope);
       }
     }
   },
 });
+
 export default testWithCloneRecording;
 export { expect } from "@playwright/test";
 export type { Page } from "@playwright/test";

diff --git a/packages/e2e-tests/tests/breakpoints-01.test.ts b/packages/e2e-tests/tests/breakpoints-01.test.ts
@@ -2,7 +2,7 @@ import { openDevToolsTab, startTest } from "../helpers";
 import { executeAndVerifyTerminalExpression } from "../helpers/console-panel";
 import { resumeToLine, rewindToLine } from "../helpers/pause-information-panel";
 import { addBreakpoint } from "../helpers/source-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_basic.html" });
 

diff --git a/packages/e2e-tests/tests/breakpoints-02.test.ts b/packages/e2e-tests/tests/breakpoints-02.test.ts
@@ -2,7 +2,7 @@ import { openDevToolsTab, startTest } from "../helpers";
 import { executeAndVerifyTerminalExpression } from "../helpers/console-panel";
 import { rewindToLine } from "../helpers/pause-information-panel";
 import { addBreakpoint } from "../helpers/source-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_basic.html" });
 

diff --git a/packages/e2e-tests/tests/breakpoints-03.test.ts b/packages/e2e-tests/tests/breakpoints-03.test.ts
@@ -2,7 +2,7 @@ import { openDevToolsTab, startTest } from "../helpers";
 import { executeAndVerifyTerminalExpression } from "../helpers/console-panel";
 import { resumeToLine, rewind, rewindToLine } from "../helpers/pause-information-panel";
 import { addBreakpoint, removeBreakpoint } from "../helpers/source-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_basic.html" });
 

diff --git a/packages/e2e-tests/tests/breakpoints-04.test.ts b/packages/e2e-tests/tests/breakpoints-04.test.ts
@@ -1,7 +1,7 @@
 import { openDevToolsTab, startTest } from "../helpers";
 import { resumeToLine, rewindToLine } from "../helpers/pause-information-panel";
 import { addBreakpoint } from "../helpers/source-panel";
-import test, { Page } from "../testFixtureCloneRecording";
+import test, { Page } from "../testFixture";
 
 test.use({ exampleKey: "doc_control_flow.html" });
 

diff --git a/packages/e2e-tests/tests/breakpoints-05.test.ts b/packages/e2e-tests/tests/breakpoints-05.test.ts
@@ -5,7 +5,7 @@ import {
   rewindToLine,
 } from "../helpers/pause-information-panel";
 import { addBreakpoint, removeBreakpoint } from "../helpers/source-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_debugger_statements.html" });
 

diff --git a/packages/e2e-tests/tests/breakpoints-06.test.ts b/packages/e2e-tests/tests/breakpoints-06.test.ts
@@ -1,7 +1,7 @@
 import { openDevToolsTab, startTest } from "../helpers";
 import { findConsoleMessage } from "../helpers/console-panel";
 import { addLogpoint } from "../helpers/source-panel";
-import test, { Page, expect } from "../testFixtureCloneRecording";
+import test, { Page, expect } from "../testFixture";
 
 async function checkMessageLocation(page: Page, text: string, location: string) {
   const message = await findConsoleMessage(page, text, "log-point");

diff --git a/packages/e2e-tests/tests/breakpoints-07.test.ts b/packages/e2e-tests/tests/breakpoints-07.test.ts
@@ -16,7 +16,7 @@ import {
   waitForLogpoint,
 } from "../helpers/source-panel";
 import { debugPrint } from "../helpers/utils";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_navigate.html" });
 

diff --git a/packages/e2e-tests/tests/breakpoints-08.test.ts b/packages/e2e-tests/tests/breakpoints-08.test.ts
@@ -6,7 +6,7 @@ import {
   togglePoint,
 } from "../helpers/pause-information-panel";
 import { addBreakpoint, removeBreakpoint } from "../helpers/source-panel";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "doc_navigate.html" });
 

diff --git a/packages/e2e-tests/tests/console-expressions-01.test.ts b/packages/e2e-tests/tests/console-expressions-01.test.ts
@@ -10,7 +10,7 @@ import {
   warpToMessage,
 } from "../helpers/console-panel";
 import { selectFrame, verifyFramesCount } from "../helpers/pause-information-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_async.html" });
 

diff --git a/packages/e2e-tests/tests/console_async_eval.test.ts b/packages/e2e-tests/tests/console_async_eval.test.ts
@@ -7,7 +7,7 @@ import {
 } from "../helpers/console-panel";
 import { selectFrame, verifyFramesCount } from "../helpers/pause-information-panel";
 import { addLogpoint } from "../helpers/source-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_async.html" });
 

diff --git a/packages/e2e-tests/tests/console_dock.test.ts b/packages/e2e-tests/tests/console_dock.test.ts
@@ -4,7 +4,7 @@ import {
   verifyToolboxLayout,
   verifyToolboxLayoutOptions,
 } from "../helpers/layout";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_basic.html" });
 

diff --git a/packages/e2e-tests/tests/console_errors.test.ts b/packages/e2e-tests/tests/console_errors.test.ts
@@ -5,7 +5,7 @@ import {
   toggleSideFilters,
   verifyConsoleMessage,
 } from "../helpers/console-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_exceptions_bundle.html" });
 

diff --git a/packages/e2e-tests/tests/console_eval.test.ts b/packages/e2e-tests/tests/console_eval.test.ts
@@ -4,7 +4,7 @@ import {
   openConsolePanel,
   warpToMessage,
 } from "../helpers/console-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_basic.html" });
 

diff --git a/packages/e2e-tests/tests/console_warp-01.test.ts b/packages/e2e-tests/tests/console_warp-01.test.ts
@@ -10,7 +10,7 @@ import {
   rewindToLine,
 } from "../helpers/pause-information-panel";
 import { addBreakpoint } from "../helpers/source-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_error.html" });
 

diff --git a/packages/e2e-tests/tests/console_warp-02.test.ts b/packages/e2e-tests/tests/console_warp-02.test.ts
@@ -7,7 +7,7 @@ import {
   warpToMessage,
 } from "../helpers/console-panel";
 import { reverseStepOverToLine, stepOverToLine } from "../helpers/pause-information-panel";
-import test from "../testFixtureCloneRecording";
+import test from "../testFixture";
 
 test.use({ exampleKey: "doc_rr_logs.html" });
 

diff --git a/packages/e2e-tests/tests/cypress-01_basic-panel.test.ts b/packages/e2e-tests/tests/cypress-01_basic-panel.test.ts
@@ -19,7 +19,7 @@ import {
   openCypressTestPanel,
 } from "../helpers/testsuites";
 import { waitFor } from "../helpers/utils";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "flake/adding-spec.ts" });
 

diff --git a/packages/e2e-tests/tests/cypress-02_test-step-timelines.test.ts b/packages/e2e-tests/tests/cypress-02_test-step-timelines.test.ts
@@ -8,7 +8,7 @@ import {
 } from "../helpers/testsuites";
 import { getTimelineCurrentHoverPercent, getTimelineCurrentPercent } from "../helpers/timeline";
 import { waitFor } from "../helpers/utils";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "flake/adding-spec.ts" });
 

diff --git a/packages/e2e-tests/tests/cypress-03_test-step-interactions.test.ts b/packages/e2e-tests/tests/cypress-03_test-step-interactions.test.ts
@@ -12,7 +12,7 @@ import {
 } from "../helpers/testsuites";
 import { waitForTimelineAdvanced } from "../helpers/timeline";
 import { delay, getByTestName, waitFor } from "../helpers/utils";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "flake/adding-spec.ts" });
 

diff --git a/packages/e2e-tests/tests/cypress-04_menu-commands.test.ts b/packages/e2e-tests/tests/cypress-04_menu-commands.test.ts
@@ -9,7 +9,7 @@ import {
 } from "../helpers/testsuites";
 import { getTimelineCurrentPercent, waitForTimelineAdvanced } from "../helpers/timeline";
 import { getByTestName, waitFor } from "../helpers/utils";
-import test, { expect } from "../testFixtureCloneRecording";
+import test, { expect } from "../testFixture";
 
 test.use({ exampleKey: "cypress-realworld/bankaccounts.spec.js" });