Simplify test results data (#70)

* removes 'protocol' naming scheme

and removes IPC architecture for reading test plans from files

* removes protocol option from cli

* updates command output from test runners

* updates snapshots

* fix types and regenerate snapshots

* moves testId property into parent TestPlanResult object

* updates snapshots

src/data/types.js

@@ -70,22 +70,36 @@
  * @property {string} name name of the test plan, defaults to 'unknown'
  * @property {AriaATCIData.Log[]} log debug messages emitted during execution of test plan
  * @property {object[]} tests
+ * @property {string} tests[].id id of a test in a test plan
  * @property {string} tests[].filepath filepath of file describing the test in the test plan
  * @property {AriaATCIData.Log[]} tests[].log subset of log emitted during this single test
- * @property {AriaATCIData.TestResult[]} tests[].results
+ * @property {AriaATCIData.TestResultOutput[]} tests[].results
  */
 
 /**
  * Result from a single test in a test plan.
  * @typedef AriaATCIData.TestResult
- * @property {number} testId numeric id of a test in a test plan
+ * @property {string} testId id of a test in a test plan
+ * @property {number} presentationNumber numeric id of a test in a test plan
  * @property {object[]} commands input commands and the speech emitted
  * @property {string} commands[].command id of input command sent to system
- * @property {string} [commands[].output] speech emitted
+ * @property {string} [commands[].response] speech emitted
  * @property {string[]} [commands[].errors] errors that occured while during command
+ * @property {object[]} commands[].assertions
+ * @property {string} commands[].assertions[].expectation
+ * @property {"pass"|"fail"|null} commands[].assertions[].verdict
+ * @property {Record<string, string>} capabilities Information about the system under test
+ */
+
+/**
+ * Result from a single test in a test plan.
+ * @typedef AriaATCIData.TestResultOutput
+ * @property {object[]} commands input commands and the speech emitted
+ * @property {string} commands[].command id of input command sent to system
+ * @property {string} [commands[].response] speech emitted
+ * @property {string[]} [commands[].errors] errors that occured while during command
+ * @property {object[]} commands[].assertions
+ * @property {string} commands[].assertions[].expectation
+ * @property {"pass"|"fail"|null} commands[].assertions[].verdict
  * @property {Record<string, string>} capabilities Information about the system under test
- * @property {object[]} results permutation of input commands and assertions passing or not passing
- * @property {string} results[].command id of input command sent to system
- * @property {string} results[].expectation description of expected assertion
- * @property {boolean} results[].pass did command pass or not pass expectation
  */

src/host/cli-run-plan.js

@@ -70,11 +70,6 @@ export const builder = (args = yargs) =>
         describe: 'Directory "plan-files" are relative to',
         default: '.',
       },
-      'plan-protocol': {
-        choices: ['fork', 'developer'],
-        default: 'fork',
-        hidden: true,
-      },
       'web-driver-url': {
         coerce(arg) {
           return new URL(arg);
@@ -195,19 +190,17 @@ function mainLoggerMiddleware(argv) {
 }
 
 function mainTestPlanMiddleware(argv) {
-  const { log, testsMatch: testPattern, planProtocol, planWorkingdir, planFiles } = argv;
+  const { log, testsMatch: testPattern, planWorkingdir, planFiles } = argv;
 
   if (!planFiles || planFiles.length === 0) {
-    throw new Error(
-      `'--plan-protocol ${planProtocol}' requires 'plan-files' argument to not be empty`
-    );
+    throw new Error(`'plan-files' argument can not be empty`);
   }
 
   const planInput = {
     workingdir: planWorkingdir,
     files: planFiles,
   };
-  const planOptions = { log, testPattern, protocol: planProtocol };
+  const planOptions = { log, testPattern };
 
   argv.plans = plansFrom(planInput, planOptions);
 }

src/host/main.js

@@ -142,7 +142,7 @@ export async function hostMain(options) {
           ...callbackBody,
           capabilities,
           status: 'COMPLETED',
-          responses: commands.map(({ output }) => output),
+          responses: commands.map(({ response }) => response),
         });
 
         plan = addTestResultToTestPlan(plan, test.filepath, result);

src/host/messages.js

@@ -43,8 +43,8 @@ export const HOST_TEMPLATES = {
   [HostMessage.START]: () => `Starting...`,
   [HostMessage.UNCAUGHT_ERROR]: ({ error }) => `Uncaught error: ${error.message}`,
   [HostMessage.WILL_STOP]: () => `Stopping...`,
-  [HostMessage.PLAN_READ]: ({ name, source, tests, files }) =>
-    `Plan '${name}' with ${tests.length} tests and ${files.length} files read from source '${source}'.`,
+  [HostMessage.PLAN_READ]: ({ name, tests, files }) =>
+    `Plan '${name}' with ${tests.length} tests and ${files.length} files read.`,
   [HostMessage.START_SERVER]: () => `Starting reference server.`,
   [HostMessage.SERVER_LISTENING]: ({ url }) => `Reference server listening on '${url}'.`,
   [HostMessage.STOP_SERVER]: () => `Stopping reference server.`,

src/host/plan-from.js

@@ -5,14 +5,10 @@
  * @module host
  */
 
-import * as child_process from 'child_process';
 import * as path from 'path';
-import { fileURLToPath } from 'url';
 
 import { compileGlob } from '../shared/file-glob.js';
 import { createHost } from '../shared/file-record.js';
-import { iterateEmitter } from '../shared/iterate-emitter.js';
-import { processExited, collectProcessPipe } from '../shared/process-util.js';
 
 import { HostMessage } from './messages.js';
 import { blankTestPlan, addFileToTestPlan, addTestToTestPlan } from './plan-object.js';
@@ -47,60 +43,16 @@ function planSelectTests(plan, { pattern = '{,**/}test*' }) {
   return plan;
 }
 
-async function planFromCommandFork({ workingdir, files }) {
-  const fork = child_process.fork(
-    path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../bin/host.js'),
-    ['read-plan', '--protocol', 'fork', ...files],
-    {
-      cwd: workingdir,
-      stdio: 'pipe',
-      serialization: 'advanced',
-    }
-  );
-
-  const stdoutJob = collectProcessPipe(fork.stdout);
-  const stderrJob = collectProcessPipe(fork.stderr);
-  const exited = processExited(fork);
-  try {
-    for await (const message of iterateEmitter(fork, 'message', 'exit', 'error')) {
-      if (message.type === 'record') {
-        await stdoutJob.cancel();
-        await stderrJob.cancel();
-        return { ...planFromRecord(message.data), source: 'fork' };
-      }
-    }
-    throw new Error(
-      `did not receive record
-stdout:
-${await stdoutJob.cancel()}
-stderr:
-${await stderrJob.cancel()}`
-    );
-  } finally {
-    await exited;
-  }
-}
-
-planFromCommandFork.protocolName = 'fork';
-
-async function planFromDeveloperInterface({ workingdir, files }) {
+async function planFrom({ workingdir, files }) {
   const host = createHost();
   const glob = files.length === 1 ? files[0] : `{${files.join(',')}}`;
   const record = await host.read(workingdir, { glob });
-  return { ...planFromRecord(record), source: 'developer' };
+  return { ...planFromRecord(record) };
 }
 
-planFromDeveloperInterface.protocolName = 'developer';
-
-const PLAN_PROTOCOL = {
-  fork: planFromCommandFork,
-  developer: planFromDeveloperInterface,
-};
-
-async function planFromFiles({ workingdir, files }, { protocol = 'fork' }) {
+async function planFromFiles({ workingdir, files }) {
   try {
-    const activeProtocol = PLAN_PROTOCOL[protocol];
-    return await activeProtocol({ workingdir, files });
+    return await planFrom({ workingdir, files });
   } catch (error) {
     throw Object.assign(new Error('could not load files'), { error });
   }
@@ -111,16 +63,12 @@ async function planFromFiles({ workingdir, files }, { protocol = 'fork' }) {
  * @param {string} target.workingdir
  * @param {string[]} target.files
  * @param {object} [options]
- * @param {'fork' | 'developer'} [options.protocol]
  * @param {string} [options.testPattern]
  * @param {AriaATCIHost.Log} [options.log]
  * @returns {AsyncGenerator<AriaATCIHost.TestPlan>}
  */
-export async function* plansFrom(
-  { workingdir, files },
-  { log = () => {}, protocol, testPattern } = {}
-) {
-  const plan = await planFromFiles({ workingdir, files }, { protocol });
+export async function* plansFrom({ workingdir, files }, { log = () => {}, testPattern } = {}) {
+  const plan = await planFromFiles({ workingdir, files });
   const testPlan = planSelectTests(plan, { pattern: testPattern });
   log(HostMessage.PLAN_READ, testPlan);
   yield testPlan;

src/host/plan-object.js

@@ -14,7 +14,6 @@ import * as arrayUtil from '../shared/array-util.js';
 export function blankTestPlan(name) {
   return {
     name,
-    source: 'unknown',
     serverOptions: {
       baseUrl: {
         protocol: 'unknown',
@@ -59,7 +58,7 @@ export function addTestToTestPlan(testPlan, filepath) {
     testPlan.files.find(file => file.name === filepath),
     () => `File ${filepath} does not exist in test plan.`
   );
-  return { ...testPlan, tests: [...testPlan.tests, { filepath, log: [], results: [] }] };
+  return { ...testPlan, tests: [...testPlan.tests, { filepath, id: '', log: [], results: [] }] };
 }
 
 /**
@@ -95,9 +94,14 @@ export function addTestLogToTestPlan(testPlan, { filepath: testFilepath }) {
  */
 export function addTestResultToTestPlan(testPlan, testFilepath, result) {
   const test = testPlan.tests.find(({ filepath }) => filepath === testFilepath);
+  const { testId, presentationNumber, ...resultOutput } = result;
   return {
     ...testPlan,
-    tests: arrayUtil.replace(testPlan.tests, test, { ...test, results: [...test.results, result] }),
+    tests: arrayUtil.replace(testPlan.tests, test, {
+      ...test,
+      id: testId,
+      results: [...test.results, resultOutput],
+    }),
   };
 }
 

src/host/tests/plan-from.js

@@ -41,7 +41,6 @@ test('plansFrom', async t => {
               },
               {
                 testPattern,
-                protocol,
                 log,
               }
             )) {