Exclude 0-level priority assertions in the collection form

lib/data/parse-test-csv-row.js

@@ -193,45 +193,30 @@ function parseTestCSVRowV2({ tests, assertions, scripts, commands }) {
 
     for (const command of commands) {
       testsParsed.forEach(test => {
-        test.assertions?.forEach(assertion => {
-          if (
-            command.testId === test.testId &&
-            test.target.ats.some(
-              at => at.key === key && at.settings === (command.settings || 'defaultMode')
-            )
-          ) {
-            if (!assertion.commandInfo) assertion.commandInfo = {};
-            if (!assertion.commandInfo[key]) assertion.commandInfo[key] = [];
-
-            const commandInfo = {
-              testId: command.testId,
-              command: command.command,
-              settings: command.settings || 'defaultMode',
-              presentationNumber: Number(command.presentationNumber),
-            };
-
-            // assertion level commandInfo, useful for getting assertionExceptions
-            const assertionCommandInfoExists = assertion.commandInfo[key].find(each =>
-              findCommandInfo(each, commandInfo)
-            );
-            if (!assertionCommandInfoExists) {
-              const assertionCommandInfo = {
-                ...commandInfo,
-                assertionExceptions: command.assertionExceptions,
-              };
-              assertion.commandInfo[key].push(assertionCommandInfo);
-            }
+        if (
+          command.testId === test.testId &&
+          test.target.ats.some(
+            at => at.key === key && at.settings === (command.settings || 'defaultMode')
+          )
+        ) {
+          const commandInfo = {
+            testId: command.testId,
+            command: command.command,
+            settings: command.settings || 'defaultMode',
+            presentationNumber: Number(command.presentationNumber),
+            assertionExceptions: command.assertionExceptions,
+          };
 
-            // Test level commandInfo, useful for getting review level information
-            if (!test.commandsInfo) test.commandsInfo = {};
-            if (!test.commandsInfo[key]) test.commandsInfo[key] = [];
-            const testCommandInfoExists = test.commandsInfo[key].find(each =>
-              findCommandInfo(each, commandInfo)
-            );
+          // Test level commandInfo, useful for getting assertionExceptions and
+          // sort order in review and test pages
+          if (!test.commandsInfo) test.commandsInfo = {};
+          if (!test.commandsInfo[key]) test.commandsInfo[key] = [];
+          const testCommandInfoExists = test.commandsInfo[key].find(each =>
+            findCommandInfo(each, commandInfo)
+          );
 
-            if (!testCommandInfoExists) test.commandsInfo[key].push(commandInfo);
-          }
-        });
+          if (!testCommandInfoExists) test.commandsInfo[key].push(commandInfo);
+        }
       });
     }
   };

lib/data/process-test-directory.js

@@ -215,7 +215,7 @@ const processTestDirectory = async ({ directory, args = {} }) => {
   const validCommandKeys = /^(?:testId|command|settings|assertionExceptions|presentationNumber)$/;
   const numericKeyFormat = /^_(\d+)$/;
   const idFormat = /^[A-Za-z0-9-]+$/;
-  const assertionsExceptionsFormat = /^([0123]:[a-zA-Z-]+\s*)+$/;
+  const assertionsExceptionsFormat = /^([0123]:[a-zA-Z-\d]+\s*)+$/;
   const settingsFormat = /^[A-Za-z0-9-\s]+$/;
   function validateCommandsKeys(row) {
     // example header:

scripts/test-reviewer.mjs

@@ -315,7 +315,6 @@ fse.readdirSync(testsBuildDirectory).forEach(function (directory) {
                       priority: getPriorityString(a[0]),
                       assertionPhrase: 'N/A',
                       assertionStatement: a[1],
-                      commandInfo: null,
                     };
                   }
 
@@ -326,7 +325,6 @@ fse.readdirSync(testsBuildDirectory).forEach(function (directory) {
                     assertionPhrase: a.assertionPhrase,
                     assertionStatement:
                       a.tokenizedAssertionStatements?.[at.key] || a.assertionStatement,
-                    commandInfo: a.commandInfo,
                   };
                 })
               : undefined;
@@ -368,33 +366,22 @@ fse.readdirSync(testsBuildDirectory).forEach(function (directory) {
                   let priority = assertion.priority;
 
                   // Check to see if there is any command info exceptions for current at key
-                  if (assertion.commandInfo && assertion.commandInfo[at.key]) {
-                    assertion.commandInfo[at.key].forEach(commandInfoForAt => {
-                      // Reconfirm against the known testData.commandsInfo;
-                      // order should be maintained as presorted during build
-                      // process
-                      const testDataCommandInfoForAt =
-                        testData.commandsInfo[at.key][assertionForCommandIndex];
-
-                      if (
-                        commandInfoForAt.testId === task &&
-                        commandInfoForAt.command === assertionForCommand.key &&
-                        commandInfoForAt.assertionExceptions.includes(assertion.assertionId) &&
-                        commandInfoForAt.presentationNumber ===
-                          testDataCommandInfoForAt.presentationNumber
-                      ) {
-                        for (const exceptionPair of commandInfoForAt.assertionExceptions.split(
-                          ' '
-                        )) {
-                          let [exceptionPriority, exceptionAssertion] = exceptionPair.split(':');
-                          exceptionPriority = Number(exceptionPriority);
-
-                          if (assertion.assertionId === exceptionAssertion) {
-                            priority = getPriorityString(exceptionPriority);
-                          }
-                        }
+                  const foundCommandInfo = testData.commandsInfo?.[at.key]?.find(
+                    c =>
+                      c.assertionExceptions.includes(assertion.assertionId) &&
+                      c.command === assertionForCommand.key &&
+                      c.settings === assertionForCommand.settings
+                  );
+
+                  if (foundCommandInfo) {
+                    for (const exceptionPair of foundCommandInfo.assertionExceptions.split(' ')) {
+                      let [exceptionPriority, exceptionAssertion] = exceptionPair.split(':');
+                      exceptionPriority = Number(exceptionPriority);
+
+                      if (assertion.assertionId === exceptionAssertion) {
+                        priority = getPriorityString(exceptionPriority);
                       }
-                    });
+                    }
                   }
 
                   return {

tests/resources/aria-at-test-io-format.mjs

@@ -405,6 +405,8 @@ class CommandsInput {
             commands.push(command);
             commandsAndSettings.push({
               command,
+              commandId,
+              presentationNumber: Number(presentationNumber),
               settings: _atMode,
               settingsText: assistiveTech.settings?.[_atMode]?.screenText || 'default mode active',
               settingsInstructions: assistiveTech.settings?.[_atMode]?.instructions || [
@@ -767,6 +769,9 @@ class BehaviorInput {
 
     const { commandsAndSettings } = commandsInput.getCommands({ task: json.task }, mode);
 
+    // Use to determine assertionExceptions
+    const commandsInfo = json.commandsInfo?.[at.key];
+
     return new BehaviorInput({
       behavior: {
         description: titleInput.title(),
@@ -778,7 +783,31 @@ class BehaviorInput {
         setupScriptDescription: json.setup_script_description,
         setupTestPage: json.setupTestPage,
         assertionResponseQuestion: json.assertionResponseQuestion,
-        commands: commandsAndSettings,
+        commands: commandsAndSettings.map(cs => {
+          const foundCommandInfo = commandsInfo?.find(
+            c =>
+              cs.commandId === c.command &&
+              cs.presentationNumber === c.presentationNumber &&
+              cs.settings === c.settings
+          );
+          if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs;
+
+          // Only works for v2
+          let assertionExceptions = json.output_assertions.map(each => each.assertionId);
+          foundCommandInfo.assertionExceptions.split(' ').forEach(each => {
+            let [priority, assertionId] = each.split(':');
+            const index = assertionExceptions.findIndex(each => each === assertionId);
+
+            priority = Number(priority);
+            assertionExceptions[index] = priority;
+          });
+          // Preserve default priority or update with exception
+          assertionExceptions = assertionExceptions.map((each, index) =>
+            isNaN(each) ? json.output_assertions[index].priority : each
+          );
+
+          return { ...cs, assertionExceptions };
+        }),
         assertions: (json.output_assertions ? json.output_assertions : []).map(assertion => {
           // Tuple array [ priorityNumber, assertionText ]
           if (Array.isArray(assertion)) {
@@ -788,7 +817,7 @@ class BehaviorInput {
             };
           }
 
-          // { assertionId, priority, assertionStatement, assertionPhrase, refIds, commandInfo, tokenizedAssertionStatements }
+          // { assertionId, priority, assertionStatement, assertionPhrase, refIds, tokenizedAssertionStatements }
           return {
             priority: assertion.priority,
             assertion:
@@ -815,7 +844,7 @@ class BehaviorInput {
    * @param {UnexpectedInput} data.unexpectedInput
    */
   static fromCollectedTestCommandsKeysUnexpected(
-    { info, target, instructions, assertions },
+    { info, target, instructions, assertions, commands },
     { commandsInput, keysInput, unexpectedInput }
   ) {
     // v1:info.task, v2: info.testId | v1:target.mode, v2:target.at.settings
@@ -834,7 +863,28 @@ class BehaviorInput {
         specificUserInstruction: instructions.raw || instructions.instructions,
         setupScriptDescription: target.setupScript ? target.setupScript.description : '',
         setupTestPage: target.setupScript ? target.setupScript.name : undefined,
-        commands: commandsAndSettings,
+        commands: commandsAndSettings.map(cs => {
+          const foundCommandInfo = commands.find(
+            c => cs.commandId === c.id && cs.settings === c.settings
+          );
+          if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs;
+
+          // Only works for v2
+          let assertionExceptions = assertions.map(each => each.assertionId);
+          foundCommandInfo.assertionExceptions.forEach(each => {
+            let { priority, assertionId } = each;
+            const index = assertionExceptions.findIndex(each => each === assertionId);
+
+            priority = Number(priority);
+            assertionExceptions[index] = priority;
+          });
+          // Preserve default priority or update with exception
+          assertionExceptions = assertionExceptions.map((each, index) =>
+            isNaN(each) ? assertions[index].priority : each
+          );
+
+          return { ...cs, assertionExceptions };
+        }),
         assertions: assertions.map(
           ({ priority, expectation, assertionStatement, tokenizedAssertionStatements }) => {
             let assertion = tokenizedAssertionStatements
@@ -1187,6 +1237,7 @@ export class TestRunInputOutput {
               description: command.settings,
               text: command.settingsText,
               instructions: command.settingsInstructions,
+              assertionExceptions: command.assertionExceptions,
             },
             atOutput: {
               highlightRequired: false,

tests/resources/aria-at-test-run.mjs

@@ -219,7 +219,7 @@ export function instructionDocument(resultState, hooks) {
     const resultUnexpectedBehavior = resultStateCommand.unexpected;
 
     const {
-      commandSettings: { description: settings, text: settingsText },
+      commandSettings: { description: settings, text: settingsText, assertionExceptions },
     } = resultStateCommand;
 
     return {
@@ -248,7 +248,16 @@ export function instructionDocument(resultState, hooks) {
         }?`,
       },
       assertions: [
-        ...assertions.map(bind(assertionResult, commandIndex)),
+        ...assertions
+          // Ignore assertion if level 0 priority exception found for assertion's command
+          .filter((each, index) => (assertionExceptions ? assertionExceptions[index] !== 0 : each))
+          .map(each =>
+            assertionResult(
+              commandIndex,
+              each,
+              assertions.findIndex(e => e === each)
+            )
+          ),
         ...additionalAssertions.map(bind(additionalAssertionResult, commandIndex)),
       ],
       unexpectedBehaviors: {
@@ -742,10 +751,20 @@ function resultsTableDocument(state) {
       header: [
         'Test result: ',
         state.commands.some(
-          ({ assertions, additionalAssertions, unexpected }) =>
-            [...assertions, ...additionalAssertions].some(
-              ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS
-            ) || unexpected.behaviors.some(({ checked }) => checked)
+          ({
+            assertions,
+            additionalAssertions,
+            unexpected,
+            commandSettings: { assertionExceptions },
+          }) =>
+            [
+              // Ignore assertion if level 0 priority exception found for assertion's command
+              ...assertions.filter((each, index) =>
+                assertionExceptions ? assertionExceptions[index] !== 0 : each
+              ),
+              ...additionalAssertions,
+            ].some(({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS) ||
+            unexpected.behaviors.some(({ checked }) => checked)
         )
           ? 'FAIL'
           : 'PASS',
@@ -758,11 +777,20 @@ function resultsTableDocument(state) {
         details: 'Details',
       },
       commands: state.commands.map(command => {
-        const allAssertions = [...command.assertions, ...command.additionalAssertions];
+        const {
+          commandSettings: { assertionExceptions },
+        } = command;
+        const allAssertions = [
+          // Ignore assertion if level 0 priority exception found for assertion's command
+          ...command.assertions.filter((each, index) =>
+            assertionExceptions ? assertionExceptions[index] !== 0 : each
+          ),
+          ...command.additionalAssertions,
+        ];
 
         let passingAssertions = ['No passing assertions.'];
         let failingAssertions = ['No failing assertions.'];
-        let unexpectedBehaviors = ['No unexpect behaviors.'];
+        let unexpectedBehaviors = ['No unexpected behaviors.'];
 
         if (allAssertions.some(({ result }) => result === CommonResultMap.PASS)) {
           passingAssertions = allAssertions

tests/resources/at-commands.mjs

@@ -123,6 +123,7 @@ export class commandsAPI {
                 return {
                   value,
                   key,
+                  settings: mode,
                 };
               })
             );