diff --git a/lib/data/parse-test-csv-row.js b/lib/data/parse-test-csv-row.js index c0138e46e..2a337eed1 100644 --- a/lib/data/parse-test-csv-row.js +++ b/lib/data/parse-test-csv-row.js @@ -193,45 +193,30 @@ function parseTestCSVRowV2({ tests, assertions, scripts, commands }) { for (const command of commands) { testsParsed.forEach(test => { - test.assertions?.forEach(assertion => { - if ( - command.testId === test.testId && - test.target.ats.some( - at => at.key === key && at.settings === (command.settings || 'defaultMode') - ) - ) { - if (!assertion.commandInfo) assertion.commandInfo = {}; - if (!assertion.commandInfo[key]) assertion.commandInfo[key] = []; - - const commandInfo = { - testId: command.testId, - command: command.command, - settings: command.settings || 'defaultMode', - presentationNumber: Number(command.presentationNumber), - }; - - // assertion level commandInfo, useful for getting assertionExceptions - const assertionCommandInfoExists = assertion.commandInfo[key].find(each => - findCommandInfo(each, commandInfo) - ); - if (!assertionCommandInfoExists) { - const assertionCommandInfo = { - ...commandInfo, - assertionExceptions: command.assertionExceptions, - }; - assertion.commandInfo[key].push(assertionCommandInfo); - } + if ( + command.testId === test.testId && + test.target.ats.some( + at => at.key === key && at.settings === (command.settings || 'defaultMode') + ) + ) { + const commandInfo = { + testId: command.testId, + command: command.command, + settings: command.settings || 'defaultMode', + presentationNumber: Number(command.presentationNumber), + assertionExceptions: command.assertionExceptions, + }; - // Test level commandInfo, useful for getting review level information - if (!test.commandsInfo) test.commandsInfo = {}; - if (!test.commandsInfo[key]) test.commandsInfo[key] = []; - const testCommandInfoExists = test.commandsInfo[key].find(each => - findCommandInfo(each, commandInfo) - ); + // Test level commandInfo, useful for getting assertionExceptions and + // sort order in review and test pages + if (!test.commandsInfo) test.commandsInfo = {}; + if (!test.commandsInfo[key]) test.commandsInfo[key] = []; + const testCommandInfoExists = test.commandsInfo[key].find(each => + findCommandInfo(each, commandInfo) + ); - if (!testCommandInfoExists) test.commandsInfo[key].push(commandInfo); - } - }); + if (!testCommandInfoExists) test.commandsInfo[key].push(commandInfo); + } }); } }; diff --git a/lib/data/process-test-directory.js b/lib/data/process-test-directory.js index d27079037..a0c126cfa 100644 --- a/lib/data/process-test-directory.js +++ b/lib/data/process-test-directory.js @@ -215,7 +215,7 @@ const processTestDirectory = async ({ directory, args = {} }) => { const validCommandKeys = /^(?:testId|command|settings|assertionExceptions|presentationNumber)$/; const numericKeyFormat = /^_(\d+)$/; const idFormat = /^[A-Za-z0-9-]+$/; - const assertionsExceptionsFormat = /^([0123]:[a-zA-Z-]+\s*)+$/; + const assertionsExceptionsFormat = /^([0123]:[a-zA-Z-\d]+\s*)+$/; const settingsFormat = /^[A-Za-z0-9-\s]+$/; function validateCommandsKeys(row) { // example header: diff --git a/scripts/test-reviewer.mjs b/scripts/test-reviewer.mjs index b47a4a3fa..7cb63e1d4 100644 --- a/scripts/test-reviewer.mjs +++ b/scripts/test-reviewer.mjs @@ -315,7 +315,6 @@ fse.readdirSync(testsBuildDirectory).forEach(function (directory) { priority: getPriorityString(a[0]), assertionPhrase: 'N/A', assertionStatement: a[1], - commandInfo: null, }; } @@ -326,7 +325,6 @@ fse.readdirSync(testsBuildDirectory).forEach(function (directory) { assertionPhrase: a.assertionPhrase, assertionStatement: a.tokenizedAssertionStatements?.[at.key] || a.assertionStatement, - commandInfo: a.commandInfo, }; }) : undefined; @@ -368,33 +366,22 @@ fse.readdirSync(testsBuildDirectory).forEach(function (directory) { let priority = assertion.priority; // Check to see if there is any command info exceptions for current at key - if (assertion.commandInfo && assertion.commandInfo[at.key]) { - assertion.commandInfo[at.key].forEach(commandInfoForAt => { - // Reconfirm against the known testData.commandsInfo; - // order should be maintained as presorted during build - // process - const testDataCommandInfoForAt = - testData.commandsInfo[at.key][assertionForCommandIndex]; - - if ( - commandInfoForAt.testId === task && - commandInfoForAt.command === assertionForCommand.key && - commandInfoForAt.assertionExceptions.includes(assertion.assertionId) && - commandInfoForAt.presentationNumber === - testDataCommandInfoForAt.presentationNumber - ) { - for (const exceptionPair of commandInfoForAt.assertionExceptions.split( - ' ' - )) { - let [exceptionPriority, exceptionAssertion] = exceptionPair.split(':'); - exceptionPriority = Number(exceptionPriority); - - if (assertion.assertionId === exceptionAssertion) { - priority = getPriorityString(exceptionPriority); - } - } + const foundCommandInfo = testData.commandsInfo?.[at.key]?.find( + c => + c.assertionExceptions.includes(assertion.assertionId) && + c.command === assertionForCommand.key && + c.settings === assertionForCommand.settings + ); + + if (foundCommandInfo) { + for (const exceptionPair of foundCommandInfo.assertionExceptions.split(' ')) { + let [exceptionPriority, exceptionAssertion] = exceptionPair.split(':'); + exceptionPriority = Number(exceptionPriority); + + if (assertion.assertionId === exceptionAssertion) { + priority = getPriorityString(exceptionPriority); } - }); + } } return { diff --git a/tests/resources/aria-at-test-io-format.mjs b/tests/resources/aria-at-test-io-format.mjs index d967c7392..22b60873b 100644 --- a/tests/resources/aria-at-test-io-format.mjs +++ b/tests/resources/aria-at-test-io-format.mjs @@ -405,6 +405,8 @@ class CommandsInput { commands.push(command); commandsAndSettings.push({ command, + commandId, + presentationNumber: Number(presentationNumber), settings: _atMode, settingsText: assistiveTech.settings?.[_atMode]?.screenText || 'default mode active', settingsInstructions: assistiveTech.settings?.[_atMode]?.instructions || [ @@ -767,6 +769,9 @@ class BehaviorInput { const { commandsAndSettings } = commandsInput.getCommands({ task: json.task }, mode); + // Use to determine assertionExceptions + const commandsInfo = json.commandsInfo?.[at.key]; + return new BehaviorInput({ behavior: { description: titleInput.title(), @@ -778,7 +783,31 @@ class BehaviorInput { setupScriptDescription: json.setup_script_description, setupTestPage: json.setupTestPage, assertionResponseQuestion: json.assertionResponseQuestion, - commands: commandsAndSettings, + commands: commandsAndSettings.map(cs => { + const foundCommandInfo = commandsInfo?.find( + c => + cs.commandId === c.command && + cs.presentationNumber === c.presentationNumber && + cs.settings === c.settings + ); + if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs; + + // Only works for v2 + let assertionExceptions = json.output_assertions.map(each => each.assertionId); + foundCommandInfo.assertionExceptions.split(' ').forEach(each => { + let [priority, assertionId] = each.split(':'); + const index = assertionExceptions.findIndex(each => each === assertionId); + + priority = Number(priority); + assertionExceptions[index] = priority; + }); + // Preserve default priority or update with exception + assertionExceptions = assertionExceptions.map((each, index) => + isNaN(each) ? json.output_assertions[index].priority : each + ); + + return { ...cs, assertionExceptions }; + }), assertions: (json.output_assertions ? json.output_assertions : []).map(assertion => { // Tuple array [ priorityNumber, assertionText ] if (Array.isArray(assertion)) { @@ -788,7 +817,7 @@ class BehaviorInput { }; } - // { assertionId, priority, assertionStatement, assertionPhrase, refIds, commandInfo, tokenizedAssertionStatements } + // { assertionId, priority, assertionStatement, assertionPhrase, refIds, tokenizedAssertionStatements } return { priority: assertion.priority, assertion: @@ -815,7 +844,7 @@ class BehaviorInput { * @param {UnexpectedInput} data.unexpectedInput */ static fromCollectedTestCommandsKeysUnexpected( - { info, target, instructions, assertions }, + { info, target, instructions, assertions, commands }, { commandsInput, keysInput, unexpectedInput } ) { // v1:info.task, v2: info.testId | v1:target.mode, v2:target.at.settings @@ -834,7 +863,28 @@ class BehaviorInput { specificUserInstruction: instructions.raw || instructions.instructions, setupScriptDescription: target.setupScript ? target.setupScript.description : '', setupTestPage: target.setupScript ? target.setupScript.name : undefined, - commands: commandsAndSettings, + commands: commandsAndSettings.map(cs => { + const foundCommandInfo = commands.find( + c => cs.commandId === c.id && cs.settings === c.settings + ); + if (!foundCommandInfo || !foundCommandInfo.assertionExceptions) return cs; + + // Only works for v2 + let assertionExceptions = assertions.map(each => each.assertionId); + foundCommandInfo.assertionExceptions.forEach(each => { + let { priority, assertionId } = each; + const index = assertionExceptions.findIndex(each => each === assertionId); + + priority = Number(priority); + assertionExceptions[index] = priority; + }); + // Preserve default priority or update with exception + assertionExceptions = assertionExceptions.map((each, index) => + isNaN(each) ? assertions[index].priority : each + ); + + return { ...cs, assertionExceptions }; + }), assertions: assertions.map( ({ priority, expectation, assertionStatement, tokenizedAssertionStatements }) => { let assertion = tokenizedAssertionStatements @@ -1187,6 +1237,7 @@ export class TestRunInputOutput { description: command.settings, text: command.settingsText, instructions: command.settingsInstructions, + assertionExceptions: command.assertionExceptions, }, atOutput: { highlightRequired: false, diff --git a/tests/resources/aria-at-test-run.mjs b/tests/resources/aria-at-test-run.mjs index 04c7fd4c4..d1882a3c6 100644 --- a/tests/resources/aria-at-test-run.mjs +++ b/tests/resources/aria-at-test-run.mjs @@ -219,7 +219,7 @@ export function instructionDocument(resultState, hooks) { const resultUnexpectedBehavior = resultStateCommand.unexpected; const { - commandSettings: { description: settings, text: settingsText }, + commandSettings: { description: settings, text: settingsText, assertionExceptions }, } = resultStateCommand; return { @@ -248,7 +248,16 @@ export function instructionDocument(resultState, hooks) { }?`, }, assertions: [ - ...assertions.map(bind(assertionResult, commandIndex)), + ...assertions + // Ignore assertion if level 0 priority exception found for assertion's command + .filter((each, index) => (assertionExceptions ? assertionExceptions[index] !== 0 : each)) + .map(each => + assertionResult( + commandIndex, + each, + assertions.findIndex(e => e === each) + ) + ), ...additionalAssertions.map(bind(additionalAssertionResult, commandIndex)), ], unexpectedBehaviors: { @@ -742,10 +751,20 @@ function resultsTableDocument(state) { header: [ 'Test result: ', state.commands.some( - ({ assertions, additionalAssertions, unexpected }) => - [...assertions, ...additionalAssertions].some( - ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS - ) || unexpected.behaviors.some(({ checked }) => checked) + ({ + assertions, + additionalAssertions, + unexpected, + commandSettings: { assertionExceptions }, + }) => + [ + // Ignore assertion if level 0 priority exception found for assertion's command + ...assertions.filter((each, index) => + assertionExceptions ? assertionExceptions[index] !== 0 : each + ), + ...additionalAssertions, + ].some(({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS) || + unexpected.behaviors.some(({ checked }) => checked) ) ? 'FAIL' : 'PASS', @@ -758,11 +777,20 @@ function resultsTableDocument(state) { details: 'Details', }, commands: state.commands.map(command => { - const allAssertions = [...command.assertions, ...command.additionalAssertions]; + const { + commandSettings: { assertionExceptions }, + } = command; + const allAssertions = [ + // Ignore assertion if level 0 priority exception found for assertion's command + ...command.assertions.filter((each, index) => + assertionExceptions ? assertionExceptions[index] !== 0 : each + ), + ...command.additionalAssertions, + ]; let passingAssertions = ['No passing assertions.']; let failingAssertions = ['No failing assertions.']; - let unexpectedBehaviors = ['No unexpect behaviors.']; + let unexpectedBehaviors = ['No unexpected behaviors.']; if (allAssertions.some(({ result }) => result === CommonResultMap.PASS)) { passingAssertions = allAssertions diff --git a/tests/resources/at-commands.mjs b/tests/resources/at-commands.mjs index c6bbfb5a1..6bbb5b776 100644 --- a/tests/resources/at-commands.mjs +++ b/tests/resources/at-commands.mjs @@ -123,6 +123,7 @@ export class commandsAPI { return { value, key, + settings: mode, }; }) );