test(e2e): Major refactor and stabilization of e2e tests

.circleci/config.yml

@@ -93,7 +93,7 @@ jobs:
             - generate_and_store_version_and_filesystem_artifacts
   e2e-test:
     parameters:
-      suite: #stable or full
+      suite: #ci or full
         type: string
     executor: pw-focal-development
     parallelism: 7
@@ -162,7 +162,7 @@ jobs:
       - run: npx playwright@1.45.2 install #Necessary for bare ubuntu machine
       - run: |
           export $(cat src/plugins/persistence/couch/.env.ci | xargs)
-          docker-compose -f src/plugins/persistence/couch/couchdb-compose.yaml up --detach
+          docker compose -f src/plugins/persistence/couch/couchdb-compose.yaml up --detach
           sleep 3
           bash src/plugins/persistence/couch/setup-couchdb.sh
       - run: sh src/plugins/persistence/couch/replace-localstorage-with-couchdb-indexhtml.sh #Replace LocalStorage Plugin with CouchDB
@@ -253,8 +253,8 @@ workflows:
           name: node18-chrome
           node-version: lts/hydrogen
       - e2e-test:
-          name: e2e-stable
-          suite: stable
+          name: e2e-ci
+          suite: ci
       - e2e-mobile
       - visual-a11y:
           name: visual-a11y-ci

.cspell.json

@@ -482,19 +482,10 @@
     "composables",
     "countup",
     "darkmatter",
-    "Undeletes"
-  ],
-  "dictionaries": [
-    "npm",
-    "softwareTerms",
-    "node",
-    "html",
-    "css",
-    "bash",
-    "en_US",
-    "en-gb",
-    "misc"
+    "Undeletes",
+    "SSSZ"
   ],
+  "dictionaries": ["npm", "softwareTerms", "node", "html", "css", "bash", "en_US", "en-gb", "misc"],
   "ignorePaths": [
     "package.json",
     "dist/**",
@@ -505,4 +496,4 @@
     "html-test-results",
     "test-results"
   ]
-}
+}

.eslintrc.cjs

@@ -5,9 +5,8 @@ const config = {
     browser: true,
     es2024: true,
     jasmine: true,
-    node: true,
-    worker: true,
-    serviceworker: true
+    amd: true,
+    node: true
   },
   globals: {
     _: 'readonly',

.github/workflows/e2e-couchdb.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Start CouchDB Docker Container and Init with Setup Scripts
         run: |
           export $(cat src/plugins/persistence/couch/.env.ci | xargs)
-          docker-compose -f src/plugins/persistence/couch/couchdb-compose.yaml up --detach
+          docker compose -f src/plugins/persistence/couch/couchdb-compose.yaml up --detach
           sleep 3
           bash src/plugins/persistence/couch/setup-couchdb.sh
           bash src/plugins/persistence/couch/replace-localstorage-with-couchdb-indexhtml.sh

.github/workflows/e2e-flakefinder.yml

@@ -34,7 +34,7 @@ jobs:
       - run: npm ci --no-audit --progress=false
 
       - name: Run E2E Tests (Repeated 10 Times)
-        run: npm run test:e2e:stable -- --retries=0 --repeat-each=10 --max-failures=50
+        run: npm run test:e2e:ci -- --retries=0 --repeat-each=10 --max-failures=50
 
       - name: Archive test results
         if: success() || failure()

README.md

@@ -109,7 +109,7 @@ Our e2e (end-to-end), Visual, and Performance tests leverage the Playwright fram
 - **e2e Tests**: These tests are run on every commit. To run the tests locally, use:
 
   ```sh
-  npm run test:e2e:stable
+  npm run test:e2e:ci
   ```
 
 - **Visual Tests**: For running the visual test suite, use:

TESTING.md

@@ -66,8 +66,8 @@ The e2e line coverage is a bit more complex than the karma implementation. This
 1. Each e2e suite will start webpack with the ```npm run start:coverage``` command with config `webpack.coverage.mjs` and the `babel-plugin-istanbul` plugin to generate code coverage during e2e test execution using our custom [baseFixture](./baseFixtures.js). 
 1. During testcase execution, each e2e shard will generate its piece of the larger coverage suite. **This coverage file is not merged**. The raw coverage file is stored in a `.nyc_report` directory.
 1. [nyc](https://github.com/istanbuljs/nyc) converts this directory into a `lcov` file with the following command `npm run cov:e2e:report`
-1. Most of the tests are run in the '@stable' configuration and focus on chrome/ubuntu at a single resolution. This coverage is published to codecov with `npm run cov:e2e:stable:publish`.
-1. The rest of our coverage only appears when run against `@unstable` tests, persistent datastore (couchdb), non-ubuntu machines, and non-chrome browsers with the `npm run cov:e2e:full:publish` flag. Since this happens about once a day, we have leveraged codecov.io's carryforward flag to report on lines covered outside of each commit on an individual PR.
+1. Most of the tests focus on chrome/ubuntu at a single resolution. This coverage is published to codecov with `npm run cov:e2e:ci:publish`.
+1. The rest of our coverage only appears when run against persistent datastore (couchdb), non-ubuntu machines, and non-chrome browsers with the `npm run cov:e2e:full:publish` flag. Since this happens about once a day, we have leveraged codecov.io's carryforward flag to report on lines covered outside of each commit on an individual PR.
 
 
 ### Limitations in our code coverage reporting

codecov.yml

@@ -11,18 +11,18 @@ coverage:
         informational: true
   precision: 2
   round: down
-  range: '66...100'
+  range: "66...100"
 
 flags:
   unit:
     carryforward: false
-  e2e-stable:
+  e2e-ci:
     carryforward: false
   e2e-full:
     carryforward: true
 
 comment:
-  layout: 'diff,flags,files,footer'
+  layout: "diff,flags,files,footer"
   behavior: default
   require_changes: false
   show_carryforward_flags: true

e2e/.eslintrc.cjs

@@ -1,14 +1,24 @@
 /* eslint-disable no-undef */
 module.exports = {
-  extends: ['plugin:playwright/playwright-test'],
+  extends: ['plugin:playwright/recommended'],
   rules: {
-    'playwright/max-nested-describe': ['error', { max: 1 }]
+    'playwright/max-nested-describe': ['error', { max: 1 }],
+    'playwright/expect-expect': 'off'
   },
   overrides: [
     {
-      files: ['tests/visual/*.spec.js'],
+      //Apply Best Practices to externalFixtures and exampleTemplate.e2e.spec.js
+      files: [
+        'appActions.js',
+        'baseFixtures.js',
+        'pluginFixtures.js',
+        '**/exampleTemplate.e2e.spec.js'
+      ],
       rules: {
-        'playwright/no-wait-for-timeout': 'off'
+        'playwright/no-raw-locators': 'error',
+        'playwright/no-nth-methods': 'error',
+        'playwright/no-get-by-title': 'error',
+        'playwright/prefer-comparison-matcher': 'error'
       }
     }
   ]

e2e/README.md

@@ -225,14 +225,13 @@ Current list of test tags:
 |:-:|-|
 |`@mobile` | Test case or test suite is compatible with Playwright's iPad support and Open MCT's read-only mobile view (i.e. no create button).|
 |`@a11y` | Test case or test suite to execute playwright-axe accessibility checks and generate a11y reports.|
-|`@gds` | Denotes a GDS Test Case used in the VIPER Mission.|
 |`@addInit` | Initializes the browser with an injected and artificial state. Useful for loading non-default plugins. Likely will not work outside of `npm start`.|
 |`@localStorage` | Captures or generates session storage to manipulate browser state. Useful for excluding in tests which require a persistent backend (i.e. CouchDB). See [note](#utilizing-localstorage)|
 |`@snapshot` | Uses Playwright's snapshot functionality to record a copy of the DOM for direct comparison. Must be run inside of the playwright container.|
-|`@unstable` | A new test or test which is known to be flaky.|
 |`@2p` | Indicates that multiple users are involved, or multiple tabs/pages are used. Useful for testing multi-user interactivity.|
 |`@generatedata` | Indicates that a test is used to generate testdata or test the generated test data. Usually to be associated with localstorage, but this may grow over time.|
 |`@clock` | A test which modifies the clock. These have expanded out of the visual tests and into the functional tests.
+|`@framework` | A test for open mct e2e capabilities. This is primarily to ensure we don't break projects which depend on sourcing this project's fixtures like appActions.js.
 
 ### Continuous Integration
 
@@ -248,7 +247,7 @@ Our CI environment consists of 3 main modes of operation:
 
 CircleCI
 
-- Stable e2e tests against ubuntu and chrome
+- e2e tests against ubuntu and chrome
 - Performance tests against ubuntu and chrome
 - e2e tests are linted
 - Visual and a11y tests are run in a single resolution on the default `espresso` theme
@@ -287,18 +286,6 @@ So for every commit, Playwright is effectively running 4 x 2 concurrent browserc
 
 At the same time, we don't want to waste CI resources on parallel runs, so we've configured each shard to fail after 5 test failures. Test failure logs are recorded and stored to allow fast triage.
 
-#### Test Promotion
-
-In order to maintain fast and reliable feedback, tests go through a promotion process. All new test cases or test suites must be labeled with the `@unstable` annotation. The Open MCT dev team runs these unstable tests in our private repos to ensure they work downstream and are reliable.
-
-- To run the stable tests, use the `npm run test:e2e:stable` command.
-- To run the new and flaky tests, use the `npm run test:e2e:unstable` command.
-
-A testcase and testsuite are to be unmarked as @unstable when:
-
-1. They run as part of "full" run 5 times without failure.
-2. They've been by a Open MCT Developer 5 times in the closed source repo without failure.
-
 ### Cross-browser and Cross-operating system
 
 #### **What's supported:**
@@ -380,8 +367,7 @@ By adhering to this principle, we can create tests that are both robust and refl
   1. Avoid creating locator aliases. This likely means that you're compensating for a bad locator. Improve the application instead.
   1. Leverage `await page.goto('./', { waitUntil: 'domcontentloaded' });` instead of `{ waitUntil: 'networkidle' }`. Tests run against deployments with websockets often have issues with the networkidle detection.
 
-#### How to make tests faster and more resilient
-
+#### How to make tests faster and more resilient to application changes
   1. Avoid app interaction when possible. The best way of doing this is to navigate directly by URL:
 
   ```js
@@ -396,6 +382,16 @@ By adhering to this principle, we can create tests that are both robust and refl
     - Initial navigation should _almost_ always use the `{ waitUntil: 'domcontentloaded' }` option.
   1. Avoid repeated setup to test a single assertion. Write longer tests with multiple soft assertions.
   This ensures that your changes will be picked up with large refactors.
+  1. Use [user-facing locators](https://playwright.dev/docs/best-practices#use-locators) (Now a eslint rule!)
+
+  ```js
+  page.getByRole('button', { name: 'Create' } )
+  ```
+  Instead of 
+  ```js
+  page.locator('.c-create-button')
+  ```
+  Note: `page.locator()` can be used in performance tests as xk6-browser does not yet support the new `page.getBy` pattern and css lookups can be [1.5x faster](https://serpapi.com/blog/css-selectors-faster-than-getbyrole-playwright/)
 
 ##### Utilizing LocalStorage
 
@@ -448,6 +444,7 @@ By adhering to this principle, we can create tests that are both robust and refl
 - Use Open MCT's fixed-time mode unless explicitly testing realtime clock
 - Employ the `createExampleTelemetryObject` appAction to source telemetry and specify a `name` to avoid autogenerated names.
 - Avoid creating objects with a time component like timers and clocks.
+- Utilize the playwright clock() API. See @clock Annotations for examples.
 
 5. **Hide the Tree and Inspector**: Generally, your test will not require comparisons involving the tree and inspector. These aspects are covered in component-specific tests (explained below). To exclude them from the comparison by default, navigate to the root of the main view with the tree and inspector hidden:
     - `await page.goto('./#/browse/mine?hideTree=true&hideInspector=true')`
@@ -493,29 +490,25 @@ For best practices with regards to mocking network responses, see our [couchdb.e
 The following contains a list of tips and tricks which don't exactly fit into a FAQ or Best Practices doc.
 
 - (Advanced) Overriding the Browser's Clock
-It is possible to override the browser's clock in order to control time-based elements. Since this can cause unwanted behavior (i.e. Tree not rendering), only use this sparingly. To do this, use the `overrideClock` fixture as such:
+It is possible to override the browser's clock in order to control time-based elements. Since this can cause unwanted behavior -- i.e. Tree not rendering -- only use this sparingly. Use the `page.clock()` API as such:
 
 ```js
 import { test, expect } from '../../pluginFixtures.js';
 
-test.describe('foo test suite', () => {
-
-  // All subsequent tests in this suite will override the clock
-  test.use({
-    clockOptions: {
-      now: 1732413600000, // A timestamp given as milliseconds since the epoch
-      shouldAdvanceTime: true // Should the clock tick?
-    }
+test.describe('foo test suite @clock', () => {
+  test.beforeEach(async ({ page }) => {
+    //Set clock time
+    await page.clock.install({ time: MISSION_TIME });
+    await page.clock.resume();
+    //Navigate to page with new clock
+    await page.goto('./', { waitUntil: 'domcontentloaded' });
   });
 
-  test('bar test', async ({ page }) => {
-    // ...
+  test('bar here', async ({ page }) => {
+    /// ...
   });
-});
   ```
 
-  More info and options for `overrideClock` can be found in [baseFixtures.js](baseFixtures.js)
-
 - Working with multiple pages
 There are instances where multiple browser pages will needed to verify multi-page or multi-tab application behavior. Make sure to use the `@2p` annotation as well as name each page appropriately: i.e. `page1` and `page2` or `tab1` and `tab2` depending on the intended use case. Generally pages should be used unless testing `sharedWorker` code, specifically.