Support 0-level assertion priorities on TestRun page (#863)

* Update client/resources

* Update graphql-schema.js and testsResolver

* Update assertionResultsResolver.js

* Update import-tests script to include assertion.assertionExceptions (0 level assertions will be considered internally as 'EXCLUDE')

* Update getMetrics and queries to be aware of 'excluded' assertions for commands

* Update import branch for testing

* Update import branch for testing

* Remove 'exclude' property check reliance on frontend

* Revert graphql fragment usage

* Fix missing renderableContent in anon viewer query

* Rename instances of Command.settings to Command.atOperatingMode in graphql-schema.js

* Include migration to update existing test plan versions using the v2 test
format version to also include (an empty) assertionExceptions so hashes will remain in sync; include utility introduced in #880

* Move atOperatingMode setting to TestsService, getTests

* Missing comma

* Update comments with reasoning for filtering out assertions that can possibly have no ids

* Update convertAssertionPriority usage

* Revert test branch being used back to master

* Break out in-line filter into function sufficiently support readability purposes

---------

Co-authored-by: Stalgia Grigg <stalgia@bocoup.com>

client/components/TestRenderer/index.jsx

@@ -248,7 +248,7 @@ const TestRenderer = ({
         }
 
         for (let i = 0; i < scenarioResults.length; i++) {
-            const {
+            let {
                 output,
                 assertionResults,
                 unexpectedBehaviors,
@@ -259,15 +259,23 @@ const TestRenderer = ({
             if (output) commands[i].atOutput.value = output;
             commands[i].atOutput.highlightRequired = highlightRequired;
 
+            // Required because assertionResults can now be returned without an id if there is a 0-priority exception
+            // applied
+            assertionResults = assertionResults.filter(el => !!el.id);
+
             for (let j = 0; j < assertionResults.length; j++) {
-                const assertionResult = assertionResults[j];
-                const { highlightRequired } = assertionResult;
+                const { passed, highlightRequired, assertion } =
+                    assertionResults[j];
 
-                commands[i].assertions[j].result = assertionResult.passed
+                let assertionForCommandIndex = commands[i].assertions.findIndex(
+                    ({ description }) => description === assertion?.text
+                );
+                commands[i].assertions[assertionForCommandIndex].result = passed
                     ? 'pass'
                     : 'fail';
-
-                commands[i].assertions[j].highlightRequired = highlightRequired;
+                commands[i].assertions[
+                    assertionForCommandIndex
+                ].highlightRequired = highlightRequired;
             }
 
             if (unexpectedBehaviors && unexpectedBehaviors.length) {

client/components/TestRun/index.jsx

@@ -402,9 +402,12 @@ const TestRun = () => {
 
             // process assertion results
             for (let j = 0; j < assertions.length; j++) {
-                const { result, highlightRequired } = assertions[j];
+                const { description, result, highlightRequired } =
+                    assertions[j];
                 const assertionResult = {
-                    ...scenarioResult.assertionResults[j],
+                    ...scenarioResult.assertionResults.find(
+                        ({ assertion: { text } }) => text === description
+                    ),
                     passed: result === 'pass'
                 };
                 assertionResults.push(
@@ -656,10 +659,18 @@ const TestRun = () => {
                 id,
                 output: output,
                 unexpectedBehaviors: unexpectedBehaviors,
-                assertionResults: assertionResults.map(({ id, passed }) => ({
-                    id,
-                    passed
-                }))
+                assertionResults: assertionResults
+                    // All assertions are always being passed from the TestRenderer results, but
+                    // when there is a 0-priority assertion exception, an id won't be provided,
+                    // so do not include that result.
+                    // This is due to the TestRenderer still requiring the position of the
+                    // excluded assertion, but it can be removed at this point before being passed
+                    // to the server
+                    .filter(el => !!el.id)
+                    .map(({ id, passed }) => ({
+                        id,
+                        passed
+                    }))
             })
         );
 

client/components/TestRun/queries.js

@@ -523,6 +523,7 @@ export const FIND_OR_CREATE_TEST_RESULT_MUTATION = gql`
                         id
                         title
                         phase
+                        updatedAt
                         gitSha
                         testPageUrl
                         testPlan {
@@ -807,6 +808,7 @@ export const SAVE_TEST_RESULT_MUTATION = gql`
                         id
                         title
                         phase
+                        updatedAt
                         gitSha
                         testPageUrl
                         testPlan {
@@ -1091,6 +1093,7 @@ export const SUBMIT_TEST_RESULT_MUTATION = gql`
                         id
                         title
                         phase
+                        updatedAt
                         gitSha
                         testPageUrl
                         testPlan {

client/resources/aria-at-test-io-format.mjs

@@ -405,6 +405,8 @@ class CommandsInput {
             commands.push(command);
             commandsAndSettings.push({
               command,
+              commandId,
+              presentationNumber: Number(presentationNumber),
               settings: _atMode,
               settingsText: assistiveTech.settings?.[_atMode]?.screenText || 'default mode active',
               settingsInstructions: assistiveTech.settings?.[_atMode]?.instructions || [
@@ -767,6 +769,9 @@ class BehaviorInput {
 
     const { commandsAndSettings } = commandsInput.getCommands({ task: json.task }, mode);
 
+    // Use to determine assertionExceptions
+    const commandsInfo = json.commandsInfo?.[at.key];
+
     return new BehaviorInput({
       behavior: {
         description: titleInput.title(),
@@ -778,7 +783,31 @@ class BehaviorInput {
         setupScriptDescription: json.setup_script_description,
         setupTestPage: json.setupTestPage,
         assertionResponseQuestion: json.assertionResponseQuestion,
-        commands: commandsAndSettings,
+        commands: commandsAndSettings.map(cs => {
+          const foundCommandInfo = commandsInfo?.find(
+            c =>
+              cs.commandId === c.command &&
+              cs.presentationNumber === c.presentationNumber &&
+              cs.settings === c.settings
+          );
+          if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs;
+
+          // Only works for v2
+          let assertionExceptions = json.output_assertions.map(each => each.assertionId);
+          foundCommandInfo.assertionExceptions.split(' ').forEach(each => {
+            let [priority, assertionId] = each.split(':');
+            const index = assertionExceptions.findIndex(each => each === assertionId);
+
+            priority = Number(priority);
+            assertionExceptions[index] = priority;
+          });
+          // Preserve default priority or update with exception
+          assertionExceptions = assertionExceptions.map((each, index) =>
+            isNaN(each) ? json.output_assertions[index].priority : each
+          );
+
+          return { ...cs, assertionExceptions };
+        }),
         assertions: (json.output_assertions ? json.output_assertions : []).map(assertion => {
           // Tuple array [ priorityNumber, assertionText ]
           if (Array.isArray(assertion)) {
@@ -788,7 +817,7 @@ class BehaviorInput {
             };
           }
 
-          // { assertionId, priority, assertionStatement, assertionPhrase, refIds, commandInfo, tokenizedAssertionStatements }
+          // { assertionId, priority, assertionStatement, assertionPhrase, refIds, tokenizedAssertionStatements }
           return {
             priority: assertion.priority,
             assertion:
@@ -815,7 +844,7 @@ class BehaviorInput {
    * @param {UnexpectedInput} data.unexpectedInput
    */
   static fromCollectedTestCommandsKeysUnexpected(
-    { info, target, instructions, assertions },
+    { info, target, instructions, assertions, commands },
     { commandsInput, keysInput, unexpectedInput }
   ) {
     // v1:info.task, v2: info.testId | v1:target.mode, v2:target.at.settings
@@ -834,7 +863,28 @@ class BehaviorInput {
         specificUserInstruction: instructions.raw || instructions.instructions,
         setupScriptDescription: target.setupScript ? target.setupScript.description : '',
         setupTestPage: target.setupScript ? target.setupScript.name : undefined,
-        commands: commandsAndSettings,
+        commands: commandsAndSettings.map(cs => {
+          const foundCommandInfo = commands.find(
+            c => cs.commandId === c.id && cs.settings === c.settings
+          );
+          if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs;
+
+          // Only works for v2
+          let assertionExceptions = assertions.map(each => each.assertionId);
+          foundCommandInfo.assertionExceptions.forEach(each => {
+            let { priority, assertionId } = each;
+            const index = assertionExceptions.findIndex(each => each === assertionId);
+
+            priority = Number(priority);
+            assertionExceptions[index] = priority;
+          });
+          // Preserve default priority or update with exception
+          assertionExceptions = assertionExceptions.map((each, index) =>
+            isNaN(each) ? assertions[index].priority : each
+          );
+
+          return { ...cs, assertionExceptions };
+        }),
         assertions: assertions.map(
           ({ priority, expectation, assertionStatement, tokenizedAssertionStatements }) => {
             let assertion = tokenizedAssertionStatements
@@ -1187,6 +1237,7 @@ export class TestRunInputOutput {
               description: command.settings,
               text: command.settingsText,
               instructions: command.settingsInstructions,
+              assertionExceptions: command.assertionExceptions,
             },
             atOutput: {
               highlightRequired: false,

client/resources/aria-at-test-run.mjs

@@ -219,7 +219,7 @@ export function instructionDocument(resultState, hooks) {
     const resultUnexpectedBehavior = resultStateCommand.unexpected;
 
     const {
-      commandSettings: { description: settings, text: settingsText },
+      commandSettings: { description: settings, text: settingsText, assertionExceptions },
     } = resultStateCommand;
 
     return {
@@ -248,7 +248,16 @@ export function instructionDocument(resultState, hooks) {
         }?`,
       },
       assertions: [
-        ...assertions.map(bind(assertionResult, commandIndex)),
+        ...assertions
+          // Ignore assertion if level 0 priority exception found for assertion's command
+          .filter((each, index) => (assertionExceptions ? assertionExceptions[index] !== 0 : each))
+          .map(each =>
+            assertionResult(
+              commandIndex,
+              each,
+              assertions.findIndex(e => e === each)
+            )
+          ),
         ...additionalAssertions.map(bind(additionalAssertionResult, commandIndex)),
       ],
       unexpectedBehaviors: {
@@ -742,10 +751,20 @@ function resultsTableDocument(state) {
       header: [
         'Test result: ',
         state.commands.some(
-          ({ assertions, additionalAssertions, unexpected }) =>
-            [...assertions, ...additionalAssertions].some(
-              ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS
-            ) || unexpected.behaviors.some(({ checked }) => checked)
+          ({
+            assertions,
+            additionalAssertions,
+            unexpected,
+            commandSettings: { assertionExceptions },
+          }) =>
+            [
+              // Ignore assertion if level 0 priority exception found for assertion's command
+              ...assertions.filter((each, index) =>
+                assertionExceptions ? assertionExceptions[index] !== 0 : each
+              ),
+              ...additionalAssertions,
+            ].some(({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS) ||
+            unexpected.behaviors.some(({ checked }) => checked)
         )
           ? 'FAIL'
           : 'PASS',
@@ -758,11 +777,20 @@ function resultsTableDocument(state) {
         details: 'Details',
       },
       commands: state.commands.map(command => {
-        const allAssertions = [...command.assertions, ...command.additionalAssertions];
+        const {
+          commandSettings: { assertionExceptions },
+        } = command;
+        const allAssertions = [
+          // Ignore assertion if level 0 priority exception found for assertion's command
+          ...command.assertions.filter((each, index) =>
+            assertionExceptions ? assertionExceptions[index] !== 0 : each
+          ),
+          ...command.additionalAssertions,
+        ];
 
         let passingAssertions = ['No passing assertions.'];
         let failingAssertions = ['No failing assertions.'];
-        let unexpectedBehaviors = ['No unexpect behaviors.'];
+        let unexpectedBehaviors = ['No unexpected behaviors.'];
 
         if (allAssertions.some(({ result }) => result === CommonResultMap.PASS)) {
           passingAssertions = allAssertions

client/resources/at-commands.mjs

@@ -123,6 +123,7 @@ export class commandsAPI {
                 return {
                   value,
                   key,
+                  settings: mode,
                 };
               })
             );

server/graphql-schema.js

@@ -532,6 +532,14 @@ const graphqlSchema = gql`
         A human-readable version of the command, such as "Control+Alt+Down"
         """
         text: String!
+        """
+        The AT mode this command may be getting ran in, such as quickNavOn,
+        browseMode, etc.
+        The same command can be ran during the same test, but in a different
+        mode.
+        """
+        # TODO: Add link to list of known AT modes
+        atOperatingMode: String
     }
 
     """

server/migrations/20240111225130-includeMissingAssertionExceptions.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const { regenerateResultsAndRecalculateHashes } = require('./utils');
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+    async up(queryInterface, Sequelize) {
+        const updateV2TestsToIncludeEmptyAssertionExceptions =
+            async transaction => {
+                const testPlanVersions = await queryInterface.sequelize.query(
+                    `SELECT id, tests FROM "TestPlanVersion" WHERE metadata->>'testFormatVersion' = '2'`,
+                    {
+                        type: Sequelize.QueryTypes.SELECT,
+                        transaction
+                    }
+                );
+                await Promise.all(
+                    testPlanVersions.map(async ({ id, tests }) => {
+                        const updatedTests = JSON.stringify(
+                            tests.map(test => {
+                                test.assertions = test.assertions.map(
+                                    assertion => ({
+                                        ...assertion,
+                                        assertionExceptions: []
+                                    })
+                                );
+                                return test;
+                            })
+                        );
+                        await queryInterface.sequelize.query(
+                            `UPDATE "TestPlanVersion" SET tests = ? WHERE id = ?`,
+                            { replacements: [updatedTests, id], transaction }
+                        );
+                    })
+                );
+            };
+
+        return queryInterface.sequelize.transaction(async transaction => {
+            await updateV2TestsToIncludeEmptyAssertionExceptions(transaction);
+
+            // Recalculate the hashes
+            await regenerateResultsAndRecalculateHashes(
+                queryInterface,
+                transaction
+            );
+        });
+    }
+};

server/models/services/TestsService.js

@@ -36,7 +36,8 @@ const getTests = parentRecord => {
                                 id: commandKVs[0].key,
                                 text: `${commandKVs[0].value}${
                                     screenText ? ` (${screenText})` : ''
-                                }`
+                                }`,
+                                atOperatingMode: scenario.settings
                             };
                         }
                         return { id: '', text: '' };