diff --git a/client/components/BotRunTestStatusList/index.js b/client/components/BotRunTestStatusList/index.js index 4fb00a729..6d1873c50 100644 --- a/client/components/BotRunTestStatusList/index.js +++ b/client/components/BotRunTestStatusList/index.js @@ -1,13 +1,9 @@ -import React, { useEffect, useMemo, useRef, useState } from 'react'; +import React, { useMemo } from 'react'; import PropTypes from 'prop-types'; -import { - COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY, - TEST_PLAN_RUNS_TEST_RESULTS_QUERY -} from './queries'; -import { useLazyQuery, useQuery } from '@apollo/client'; +import { TEST_PLAN_RUNS_TEST_RESULTS_QUERY } from './queries'; +import { useQuery } from '@apollo/client'; import styled from '@emotion/styled'; import ReportStatusDot from '../common/ReportStatusDot'; -import { isBot } from '../../utils/automation'; const BotRunTestStatusUnorderedList = styled.ul` list-style-type: none; @@ -35,7 +31,9 @@ const BotRunTestStatusUnorderedList = styled.ul` const testCountString = (count, status) => `${count} Test${count === 1 ? '' : 's'} ${status}`; -const BotRunTestStatusList = ({ testPlanReportId, runnableTestsLength }) => { +const pollInterval = 2000; + +const BotRunTestStatusList = ({ testPlanReportId }) => { const { data: testPlanRunsQueryResult, startPolling, @@ -43,113 +41,83 @@ const BotRunTestStatusList = ({ testPlanReportId, runnableTestsLength }) => { } = useQuery(TEST_PLAN_RUNS_TEST_RESULTS_QUERY, { variables: { testPlanReportId }, fetchPolicy: 'cache-and-network', - pollInterval: 2000 + pollInterval }); - const [getCollectionJobStatus, { data: collectionJobStatusQueryResult }] = - useLazyQuery(COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY, { - fetchPolicy: 'cache-and-network' - }); - - const [collectedData, setCollectedData] = useState([]); - const requestedTestRunIds = useRef(new Set()); - - const botTestPlanRuns = useMemo(() => { - if (!testPlanRunsQueryResult?.testPlanRuns) { - return []; - } - return testPlanRunsQueryResult.testPlanRuns.filter(testPlanRun => - isBot(testPlanRun.tester) - ); - }, [testPlanRunsQueryResult?.testPlanRuns]); - - useEffect(() => { - const ids = botTestPlanRuns.map(run => run.id); - for (const id of ids) { - if (!requestedTestRunIds.current.has(id)) { - requestedTestRunIds.current.add(id); - getCollectionJobStatus({ - variables: { testPlanRunId: id } - }); - } - } - }, [botTestPlanRuns]); - - useEffect(() => { - if (collectionJobStatusQueryResult?.collectionJobByTestPlanRunId) { - const { status } = - collectionJobStatusQueryResult.collectionJobByTestPlanRunId; - setCollectedData(prev => [...prev, status]); - } - }, [collectionJobStatusQueryResult?.collectionJobByTestPlanRunId]); - - const [numTestsCompleted, numTestsQueued, numTestsCancelled] = - useMemo(() => { - const res = [0, 0, 0]; - if ( - botTestPlanRuns && - botTestPlanRuns.length && - collectedData.length === botTestPlanRuns.length - ) { - for (let i = 0; i < botTestPlanRuns.length; i++) { - const status = collectedData[i]; - res[0] += botTestPlanRuns[i].testResults.length; - switch (status) { - case 'COMPLETED': - case 'RUNNING': - case 'QUEUED': - res[1] += - runnableTestsLength - - botTestPlanRuns[i].testResults.length; - break; - case 'CANCELLED': - res[2] += - runnableTestsLength - - botTestPlanRuns[i].testResults.length; - break; - default: - break; + const { COMPLETED, ERROR, RUNNING, CANCELLED, QUEUED } = useMemo(() => { + const counter = { + COMPLETED: 0, + ERROR: 0, + RUNNING: 0, + CANCELLED: 0, + QUEUED: 0 + }; + let anyPossibleUpdates = false; + if (testPlanRunsQueryResult?.testPlanRuns) { + for (const { + collectionJob + } of testPlanRunsQueryResult.testPlanRuns) { + if (collectionJob?.testStatus) { + for (const { status } of collectionJob.testStatus) { + counter[status]++; + if (status === 'QUEUED' || status === 'RUNNING') { + anyPossibleUpdates = true; + } } } - if ( - res[0] + res[2] === - runnableTestsLength * botTestPlanRuns.length - ) { - stopPolling(); - } } - return res; - }, [testPlanRunsQueryResult, collectedData, stopPolling, startPolling]); + // it's possible that we got incomplete data on first fetch and + // stopped the polling, so restart the polling if we detect any + // possible future updates, otherwise stop. + if (anyPossibleUpdates) { + startPolling(pollInterval); + } else { + stopPolling(); + } + } + return counter; + }, [testPlanRunsQueryResult, stopPolling, startPolling]); if ( - !botTestPlanRuns || - botTestPlanRuns.length === 0 || - !collectedData || - !(collectedData.length === botTestPlanRuns.length) + !testPlanRunsQueryResult || + testPlanRunsQueryResult.testPlanRuns.length === 0 ) { return null; } return ( + {RUNNING > 0 && ( +
  • + + {testCountString(RUNNING, 'Running')} +
    • + )} + {ERROR > 0 && ( +
  • + + {testCountString(ERROR, 'Error')} +
    • + )}
  • - {testCountString(numTestsCompleted, 'Completed')} + {testCountString(COMPLETED, 'Completed')}
  • - {testCountString(numTestsQueued, 'Queued')} -
    • -
  • - - {testCountString(numTestsCancelled, 'Cancelled')} + {testCountString(QUEUED, 'Queued')}
    • + {CANCELLED > 0 && ( +
  • + + {testCountString(CANCELLED, 'Cancelled')} +
    • + )}     ); }; BotRunTestStatusList.propTypes = { - testPlanReportId: PropTypes.string.isRequired, - runnableTestsLength: PropTypes.number.isRequired + testPlanReportId: PropTypes.string.isRequired }; export default BotRunTestStatusList; diff --git a/client/components/BotRunTestStatusList/queries.js b/client/components/BotRunTestStatusList/queries.js index 048acc1aa..cd99fb2bc 100644 --- a/client/components/BotRunTestStatusList/queries.js +++ b/client/components/BotRunTestStatusList/queries.js @@ -16,15 +16,12 @@ export const TEST_PLAN_RUNS_TEST_RESULTS_QUERY = gql` } } } - } - } -`; - -export const COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY = gql` - query CollectionJobStatusByTestPlanRunId($testPlanRunId: ID!) { - collectionJobByTestPlanRunId(testPlanRunId: $testPlanRunId) { - id - status + collectionJob { + status + testStatus { + status + } + } } } `; diff --git a/client/components/common/ReportStatusDot/index.jsx b/client/components/common/ReportStatusDot/index.jsx index d369e91ad..0c44c6521 100644 --- a/client/components/common/ReportStatusDot/index.jsx +++ b/client/components/common/ReportStatusDot/index.jsx @@ -17,6 +17,15 @@ const ReportStatusDot = styled.span` background: #7c7c7c; } + &.tests-running { + border: 2px solid #1e8f37; + background: #d2d5d9; + } + + &.tests-error { + background: #e3261f; + } + &.tests-queued, &.reports-in-progress { background: #3876e8; @@ -27,7 +36,9 @@ const ReportStatusDot = styled.span` background: #2ba51c; } - &.tests-cancelled, + &.tests-cancelled { + background: #a231ff; + } &.reports-missing { background: #ce1b4c; } diff --git a/client/tests/BotRunTestStatusList.test.jsx b/client/tests/BotRunTestStatusList.test.jsx index 66b93e03d..2aec3b4c8 100644 --- a/client/tests/BotRunTestStatusList.test.jsx +++ b/client/tests/BotRunTestStatusList.test.jsx @@ -5,13 +5,11 @@ import React from 'react'; import { render, waitFor } from '@testing-library/react'; import { MockedProvider } from '@apollo/client/testing'; import BotRunTestStatusList from '../components/BotRunTestStatusList'; -import { - TEST_PLAN_RUNS_TEST_RESULTS_QUERY, - COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY -} from '../components/BotRunTestStatusList/queries'; +import { TEST_PLAN_RUNS_TEST_RESULTS_QUERY } from '../components/BotRunTestStatusList/queries'; import '@testing-library/jest-dom/extend-expect'; +import { COLLECTION_JOB_STATUS } from '../../server/util/enums'; -const getMocks = (testPlanRuns, collectionJobStatuses) => { +const getMocks = testPlanRuns => { const testPlanRunMock = { request: { query: TEST_PLAN_RUNS_TEST_RESULTS_QUERY, @@ -20,22 +18,7 @@ const getMocks = (testPlanRuns, collectionJobStatuses) => { result: { data: { testPlanRuns } } }; - const collectionJobStatusMocks = testPlanRuns.map((testRun, index) => ({ - request: { - query: COLLECTION_JOB_STATUS_BY_TEST_PLAN_RUN_ID_QUERY, - variables: { testPlanRunId: testRun.id } - }, - result: { - data: { - collectionJobByTestPlanRunId: { - status: collectionJobStatuses[index], - id: testRun.id - } - } - } - })); - - return [testPlanRunMock, ...collectionJobStatusMocks]; + return [testPlanRunMock]; }; test('correctly displays statuses for single COMPLETED test run', async () => { @@ -43,27 +26,31 @@ test('correctly displays statuses for single COMPLETED test run', async () => { { id: '0', testResults: new Array(3).fill(null), - tester: { username: 'bot' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.COMPLETED, + testStatus: [ + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.COMPLETED } + ] + } } ]; - const collectionJobStatuses = ['COMPLETED']; + const mocks = getMocks(testPlanRuns); - const mocks = getMocks(testPlanRuns, collectionJobStatuses); - - const { getByText } = render( + const screen = render( - + ); + const { getByText } = screen; + await waitFor(() => { expect(getByText('3 Tests Completed')).toBeInTheDocument(); expect(getByText('0 Tests Queued')).toBeInTheDocument(); - expect(getByText('0 Tests Cancelled')).toBeInTheDocument(); }); }); @@ -72,32 +59,34 @@ test('correctly ignores test results from a human-submitted test plan run', asyn { id: '0', testResults: new Array(2).fill(null), - tester: { username: 'bot' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.COMPLETED, + testStatus: [ + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.COMPLETED } + ] + } }, { id: '1', testResults: new Array(2).fill(null), - tester: { username: 'human' } + tester: { username: 'human' }, + collectionJob: null } ]; - const collectionJobStatuses = ['COMPLETED', 'COMPLETED']; - - const mocks = getMocks(testPlanRuns, collectionJobStatuses); + const mocks = getMocks(testPlanRuns); const { getByText } = render( - + ); await waitFor(async () => { expect(getByText('2 Tests Completed')).toBeInTheDocument(); expect(getByText('0 Tests Queued')).toBeInTheDocument(); - expect(getByText('0 Tests Cancelled')).toBeInTheDocument(); }); }); @@ -106,20 +95,23 @@ test('correctly displays statuses for CANCELLED test run', async () => { { id: '0', testResults: new Array(2).fill(null), - tester: { username: 'bot' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.CANCELLED, + testStatus: [ + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.CANCELLED } + ] + } } ]; - const collectionJobStatuses = ['CANCELLED']; - - const mocks = getMocks(testPlanRuns, collectionJobStatuses); + const mocks = getMocks(testPlanRuns); const { getByText } = render( - + ); @@ -135,35 +127,36 @@ test('correctly displays statuses for multiple RUNNING and QUEUED test runs', as { id: '0', testResults: new Array(2).fill(null), - tester: { username: 'bot' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.RUNNING, + testStatus: [ + { status: COLLECTION_JOB_STATUS.RUNNING }, + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.QUEUED } + ] + } }, { id: '1', testResults: new Array(2).fill(null), - tester: { username: 'bot' } - }, - { - id: '2', - testResults: [null], - tester: { username: 'bot' } - }, - { - id: '3', - testResults: new Array(2).fill(null), - tester: { username: 'human' } + tester: { username: 'bot' }, + collectionJob: { + status: COLLECTION_JOB_STATUS.CANCELLED, + testStatus: [ + { status: COLLECTION_JOB_STATUS.CANCELLED }, + { status: COLLECTION_JOB_STATUS.COMPLETED }, + { status: COLLECTION_JOB_STATUS.CANCELLED } + ] + } } ]; - const collectionJobStatuses = ['RUNNING', 'RUNNING', 'CANCELLED']; - - const mocks = getMocks(testPlanRuns, collectionJobStatuses); + const mocks = getMocks(testPlanRuns); const { getByText } = render( - + ); @@ -171,9 +164,10 @@ test('correctly displays statuses for multiple RUNNING and QUEUED test runs', as // Wait for the component to update // Imperfect but prevents needing to detect loading removal await setTimeout(() => { + expect(getByText('1 Test Running')).toBeInTheDocument(); expect(getByText('2 Tests Completed')).toBeInTheDocument(); expect(getByText('1 Test Queued')).toBeInTheDocument(); - expect(getByText('1 Test Cancelled')).toBeInTheDocument(); + expect(getByText('2 Tests Cancelled')).toBeInTheDocument(); }, 500); }); }); diff --git a/server/controllers/AutomationController.js b/server/controllers/AutomationController.js index 110351019..39f2ba930 100644 --- a/server/controllers/AutomationController.js +++ b/server/controllers/AutomationController.js @@ -1,7 +1,8 @@ const axios = require('axios'); const { getCollectionJobById, - updateCollectionJobById + updateCollectionJobById, + updateCollectionJobTestStatusByQuery } = require('../models/services/CollectionJobService'); const { findOrCreateTestResult @@ -17,14 +18,14 @@ const { findOrCreateBrowserVersion } = require('../models/services/BrowserService'); const { HttpQueryError } = require('apollo-server-core'); -const { COLLECTION_JOB_STATUS } = require('../util/enums'); +const { COLLECTION_JOB_STATUS, isJobStatusFinal } = require('../util/enums'); const populateData = require('../services/PopulatedData/populateData'); const { getFinalizedTestResults } = require('../models/services/TestResultReadService'); const http = require('http'); const { NO_OUTPUT_STRING } = require('../util/constants'); -const getTests = require('../models/services/TestsService'); +const runnableTestsResolver = require('../resolvers/TestPlanReport/runnableTestsResolver'); const getGraphQLContext = require('../graphql-context'); const httpAgent = new http.Agent({ family: 4 }); @@ -99,6 +100,19 @@ const updateJobStatus = async (req, res) => { ...(externalLogsUrl != null && { externalLogsUrl }) }; + // When new status is considered "final" ('COMPLETED' or 'ERROR' or 'CANCELLED') + // update any CollectionJobTestStatus children still 'QUEUED' to be 'CANCELLED' + if (isJobStatusFinal(status)) { + await updateCollectionJobTestStatusByQuery({ + where: { + collectionJobId: req.params.jobID, + status: COLLECTION_JOB_STATUS.QUEUED + }, + values: { status: COLLECTION_JOB_STATUS.CANCELLED }, + transaction: req.transaction + }); + } + const graphqlResponse = await updateCollectionJobById({ id: req.params.jobID, values: updatePayload, @@ -137,32 +151,25 @@ const getApprovedFinalizedTestResults = async (testPlanRun, context) => { return getFinalizedTestResults({ testPlanReport, context }); }; +const getTestByRowNumber = async ({ testPlanRun, testRowNumber, context }) => { + const tests = await runnableTestsResolver( + testPlanRun.testPlanReport, + null, + context + ); + return tests.find( + test => parseInt(test.rowNumber, 10) === parseInt(testRowNumber, 10) + ); +}; + const updateOrCreateTestResultWithResponses = async ({ - testRowIdentifier, + testId, testPlanRun, responses, atVersionId, browserVersionId, context }) => { - const allTestsForTestPlanVersion = await getTests( - testPlanRun.testPlanReport.testPlanVersion - ); - - const isV2 = - testPlanRun.testPlanReport.testPlanVersion.metadata - .testFormatVersion === 2; - - const testId = allTestsForTestPlanVersion.find( - test => - (!isV2 || test.at?.name === 'NVDA') && - parseInt(test.rowNumber, 10) === testRowIdentifier - )?.id; - - if (testId === undefined) { - throwNoTestFoundError(testRowIdentifier); - } - const { testResult } = await findOrCreateTestResult({ testId, testPlanRunId: testPlanRun.id, @@ -239,20 +246,20 @@ const updateOrCreateTestResultWithResponses = async ({ }; const updateJobResults = async (req, res) => { - const id = req.params.jobID; + const { jobID: id, testRowNumber } = req.params; const context = getGraphQLContext({ req }); const { transaction } = context; const { - testCsvRow, - presentationNumber, responses, + status, capabilities: { atName, atVersion: atVersionName, browserName, browserVersion: browserVersionName - } + } = {} } = req.body; + const job = await getCollectionJobById({ id, transaction }); if (!job) { throwNoJobFoundError(id); @@ -263,35 +270,65 @@ const updateJobResults = async (req, res) => { `Job with id ${id} is not running, cannot update results` ); } + if (status && !Object.values(COLLECTION_JOB_STATUS).includes(status)) { + throw new HttpQueryError(400, `Invalid status: ${status}`, true); + } + const { testPlanRun } = job; - /* TODO: Change this to use a better key based lookup system after gh-958 */ - const [at] = await getAts({ search: atName, transaction }); - const [browser] = await getBrowsers({ search: browserName, transaction }); - - const [atVersion, browserVersion] = await Promise.all([ - findOrCreateAtVersion({ - where: { atId: at.id, name: atVersionName }, - transaction - }), - findOrCreateBrowserVersion({ - where: { browserId: browser.id, name: browserVersionName }, - transaction + const testId = ( + await getTestByRowNumber({ + testPlanRun, + testRowNumber, + context }) - ]); + )?.id; + + if (testId === undefined) { + throwNoTestFoundError(testRowNumber); + } - const processedResponses = convertEmptyStringsToNoOutputMessages(responses); + // status only update, or responses were provided (default to complete) + if (status || responses) { + await updateCollectionJobTestStatusByQuery({ + where: { collectionJobId: id, testId }, + // default to completed if not specified (when results are present) + values: { status: status ?? COLLECTION_JOB_STATUS.COMPLETED }, + transaction: req.transaction + }); + } - // v1 tests store testCsvRow in rowNumber, v2 tests store presentationNumber in rowNumber - const testRowIdentifier = presentationNumber ?? testCsvRow; + // responses were provided + if (responses) { + /* TODO: Change this to use a better key based lookup system after gh-958 */ + const [at] = await getAts({ search: atName, transaction }); + const [browser] = await getBrowsers({ + search: browserName, + transaction + }); - await updateOrCreateTestResultWithResponses({ - testRowIdentifier, - responses: processedResponses, - testPlanRun: job.testPlanRun, - atVersionId: atVersion.id, - browserVersionId: browserVersion.id, - context - }); + const [atVersion, browserVersion] = await Promise.all([ + findOrCreateAtVersion({ + where: { atId: at.id, name: atVersionName }, + transaction + }), + findOrCreateBrowserVersion({ + where: { browserId: browser.id, name: browserVersionName }, + transaction + }) + ]); + + const processedResponses = + convertEmptyStringsToNoOutputMessages(responses); + + await updateOrCreateTestResultWithResponses({ + testId, + responses: processedResponses, + testPlanRun, + atVersionId: atVersion.id, + browserVersionId: browserVersion.id, + context + }); + } res.json({ success: true }); }; diff --git a/server/graphql-schema.js b/server/graphql-schema.js index 92cf9d25a..647607d99 100644 --- a/server/graphql-schema.js +++ b/server/graphql-schema.js @@ -102,6 +102,25 @@ const graphqlSchema = gql` The URL where the logs for the job can be found. """ externalLogsUrl: String + """ + An array of individual test status for every runnable test in the Job. + """ + testStatus: [CollectionJobTestStatus] + } + + """ + A status for a specific Test on a specific CollectionJob. + """ + type CollectionJobTestStatus { + """ + The test this status reflects. + """ + test: Test! + """ + The status of the test, which can be "QUEUED", "RUNNING", "COMPLETED", + "ERROR", or "CANCELLED" + """ + status: CollectionJobStatus! } type Browser { @@ -827,6 +846,10 @@ const graphqlSchema = gql` Whether the TestPlanRun was initiated by the Response Collection System """ initiatedByAutomation: Boolean! + """ + The CollectionJob related to this testPlanRun + """ + collectionJob: CollectionJob } """ diff --git a/server/migrations/20240404171101-addCollectionJobTestStatus.js b/server/migrations/20240404171101-addCollectionJobTestStatus.js new file mode 100644 index 000000000..7abf682ce --- /dev/null +++ b/server/migrations/20240404171101-addCollectionJobTestStatus.js @@ -0,0 +1,58 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + return queryInterface.sequelize.transaction(async transaction => { + await queryInterface.createTable( + 'CollectionJobTestStatus', + { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true + }, + testId: { + type: Sequelize.STRING, + allowNull: false + }, + collectionJobId: { + type: Sequelize.INTEGER, + allowNull: false, + onDelete: 'CASCADE', + onUpdate: 'CASCADE', + references: { + model: 'CollectionJob', + key: 'id' + } + }, + status: { + type: Sequelize.STRING, + allowNull: false, + defaultValue: 'QUEUED' + } + }, + { transaction } + ); + await queryInterface.addConstraint('CollectionJobTestStatus', { + type: 'unique', + name: 'CollectionJob_Test_unique', + fields: ['collectionJobId', 'testId'], + transaction + }); + }); + }, + + async down(queryInterface) { + return queryInterface.sequelize.transaction(async transaction => { + await queryInterface.removeConstraint( + 'CollectionJobTestStatus', + 'CollectionJob_Test_unique', + { transaction } + ); + await queryInterface.dropTable('CollectionJobTestStatus', { + transaction + }); + }); + } +}; diff --git a/server/models/CollectionJob.js b/server/models/CollectionJob.js index c66499a80..dc57b229d 100644 --- a/server/models/CollectionJob.js +++ b/server/models/CollectionJob.js @@ -45,6 +45,12 @@ module.exports = function (sequelize, DataTypes) { sourceKey: 'testPlanRunId', as: 'testPlanRun' }); + + Model.hasMany(models.CollectionJobTestStatus, { + as: 'testStatus', + foreignKey: 'collectionJobId', + sourceKey: 'id' + }); }; Model.QUEUED = COLLECTION_JOB_STATUS.QUEUED; diff --git a/server/models/CollectionJobTestStatus.js b/server/models/CollectionJobTestStatus.js new file mode 100644 index 000000000..88427a1dd --- /dev/null +++ b/server/models/CollectionJobTestStatus.js @@ -0,0 +1,56 @@ +const { COLLECTION_JOB_STATUS } = require('../util/enums'); + +const MODEL_NAME = 'CollectionJobTestStatus'; + +module.exports = function (sequelize, DataTypes) { + const Model = sequelize.define( + MODEL_NAME, + { + id: { + type: DataTypes.INTEGER, + allowNull: false, + primaryKey: true, + autoIncrement: true + }, + collectionJobId: { + type: DataTypes.INTEGER, + allowNull: false, + references: { + model: 'CollectionJob', + key: 'id' + } + }, + testId: { + type: DataTypes.STRING, + allowNull: null + }, + status: { + type: DataTypes.STRING, + allowNull: false, + defaultValue: COLLECTION_JOB_STATUS.QUEUED + } + }, + { + timestamps: false, + tableName: MODEL_NAME + } + ); + + Model.associate = function (models) { + Model.belongsTo(models.CollectionJob, { + foreignKey: 'collectionJobId', + targetKey: 'id', + as: 'collectionJob', + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }); + }; + + Model.QUEUED = COLLECTION_JOB_STATUS.QUEUED; + Model.RUNNING = COLLECTION_JOB_STATUS.RUNNING; + Model.COMPLETED = COLLECTION_JOB_STATUS.COMPLETED; + Model.CANCELLED = COLLECTION_JOB_STATUS.CANCELLED; + Model.ERROR = COLLECTION_JOB_STATUS.ERROR; + + return Model; +}; diff --git a/server/models/services/CollectionJobService.js b/server/models/services/CollectionJobService.js index a73dbd20e..bbe756bd9 100644 --- a/server/models/services/CollectionJobService.js +++ b/server/models/services/CollectionJobService.js @@ -1,5 +1,5 @@ const ModelService = require('./ModelService'); -const { CollectionJob } = require('../'); +const { CollectionJob, CollectionJobTestStatus } = require('../'); const { COLLECTION_JOB_ATTRIBUTES, TEST_PLAN_ATTRIBUTES, @@ -8,7 +8,8 @@ const { TEST_PLAN_VERSION_ATTRIBUTES, AT_ATTRIBUTES, BROWSER_ATTRIBUTES, - USER_ATTRIBUTES + USER_ATTRIBUTES, + COLLECTION_JOB_TEST_STATUS_ATTRIBUTES } = require('./helpers'); const { COLLECTION_JOB_STATUS } = require('../../util/enums'); const { Op } = require('sequelize'); @@ -175,10 +176,21 @@ const userAssociation = userAttributes => ({ attributes: userAttributes }); +/** + * @param {string[]} collectionJobTestStatusAttributes - attributes to be returned in the result + * @returns {{association: string, attributes: string[]}} + */ +const collectionJobTestStatusAssociation = + collectionJobTestStatusAttributes => ({ + association: 'testStatus', + attributes: collectionJobTestStatusAttributes + }); + /** * @param {object} options * @param {object} options.values - CollectionJob to be created - * @param {string[]} options.collectionJobAttributes - TestPlanRun attributes to be returned in the result + * @param {string[]} options.collectionJobAttributes - CollectionJob attributes to be returned in the result + * @param {string[]} options.collectionJobTestStatusAttributes - CollectionJobTestStatus attributes to be returned in the result * @param {string[]} options.testPlanReportAttributes - TestPlanReport attributes to be returned in the result * @param {string[]} options.testPlanVersionAttributes - TestPlanVersion attributes to be returned in the result * @param {string[]} options.testPlanAttributes - TestPlanVersion attributes to be returned in the result @@ -195,6 +207,7 @@ const createCollectionJob = async ({ testPlanReportId }, collectionJobAttributes = COLLECTION_JOB_ATTRIBUTES, + collectionJobTestStatusAttributes = COLLECTION_JOB_TEST_STATUS_ATTRIBUTES, testPlanReportAttributes = TEST_PLAN_REPORT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, testPlanVersionAttributes = TEST_PLAN_VERSION_ATTRIBUTES, @@ -222,10 +235,29 @@ const createCollectionJob = async ({ transaction }); + // create QUEUED status entries for each test in the test plan run + const context = getGraphQLContext({ req: { transaction } }); + const tests = await runnableTestsResolver( + testPlanRun.testPlanReport, + null, + context + ); + await ModelService.bulkCreate(CollectionJobTestStatus, { + valuesList: tests.map(test => ({ + testId: test.id, + collectionJobId: collectionJobResult.id, + status: COLLECTION_JOB_STATUS.QUEUED + })), + transaction + }); + return ModelService.getById(CollectionJob, { id: collectionJobResult.id, attributes: collectionJobAttributes, include: [ + collectionJobTestStatusAssociation( + collectionJobTestStatusAttributes + ), testPlanRunAssociation( testPlanRunAttributes, userAttributes, @@ -243,7 +275,8 @@ const createCollectionJob = async ({ /** * @param {object} options * @param {string} options.id - id for the CollectionJob - * @param {string[]} options.collectionJobAttributes - TestPlanRun attributes to be returned in the result + * @param {string[]} options.collectionJobAttributes - CollectionJob attributes to be returned in the result + * @param {string[]} options.collectionJobTestStatusAttributes - CollectionJobTestStatus attributes to be returned in the result * @param {string[]} options.testPlanReportAttributes - TestPlanReport attributes to be returned in the result * @param {string[]} options.testPlanVersionAttributes - TestPlanVersion attributes to be returned in the result * @param {string[]} options.testPlanAttributes - TestPlanVersion attributes to be returned in the result @@ -256,6 +289,7 @@ const createCollectionJob = async ({ const getCollectionJobById = async ({ id, collectionJobAttributes = COLLECTION_JOB_ATTRIBUTES, + collectionJobTestStatusAttributes = COLLECTION_JOB_TEST_STATUS_ATTRIBUTES, testPlanReportAttributes = TEST_PLAN_REPORT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, testPlanVersionAttributes = TEST_PLAN_VERSION_ATTRIBUTES, @@ -269,6 +303,9 @@ const getCollectionJobById = async ({ id, attributes: collectionJobAttributes, include: [ + collectionJobTestStatusAssociation( + collectionJobTestStatusAttributes + ), testPlanRunAssociation( testPlanRunAttributes, userAttributes, @@ -287,7 +324,8 @@ const getCollectionJobById = async ({ * @param {object} options * @param {string|any} options.search - use this to combine with {@param filter} to be passed to Sequelize's where clause * @param {object} options.where - use this define conditions to be passed to Sequelize's where clause - * @param {string[]} options.collectionJobAttributes - Browser attributes to be returned in the result + * @param {string[]} options.collectionJobAttributes - CollectionJob attributes to be returned in the result + * @param {string[]} options.collectionJobTestStatusAttributes - CollectionJobTestStatus attributes to be returned in the result * @param {string[]} options.testPlanReportAttributes - TestPlanReport attributes to be returned in the result * @param {string[]} options.testPlanVersionAttributes - TestPlanVersion attributes to be returned in the result * @param {string[]} options.testPlanAttributes - TestPlanVersion attributes to be returned in the result @@ -306,6 +344,7 @@ const getCollectionJobs = async ({ search, where = {}, collectionJobAttributes = COLLECTION_JOB_ATTRIBUTES, + collectionJobTestStatusAttributes = COLLECTION_JOB_TEST_STATUS_ATTRIBUTES, testPlanReportAttributes = TEST_PLAN_REPORT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, testPlanVersionAttributes = TEST_PLAN_VERSION_ATTRIBUTES, @@ -324,6 +363,9 @@ const getCollectionJobs = async ({ where, attributes: collectionJobAttributes, include: [ + collectionJobTestStatusAssociation( + collectionJobTestStatusAttributes + ), testPlanRunAssociation( testPlanRunAttributes, userAttributes, @@ -382,7 +424,8 @@ const triggerWorkflow = async (job, testIds, { transaction }) => { * @param {object} options * @param {string} options.id - id of the CollectionJob to be updated * @param {object} options.values - values to be used to update columns for the record being referenced for {@param id} - * @param {string[]} options.collectionJobAttributes - Browser attributes to be returned in the result + * @param {string[]} options.collectionJobAttributes - CollectionJob attributes to be returned in the result + * @param {string[]} options.collectionJobTestStatusAttributes - CollectionJobTestStatus attributes to be returned in the result * @param {string[]} options.testPlanReportAttributes - TestPlanReport attributes to be returned in the result * @param {string[]} options.testPlanVersionAttributes - TestPlanVersion attributes to be returned in the result * @param {string[]} options.testPlanAttributes - TestPlanVersion attributes to be returned in the result @@ -396,6 +439,7 @@ const updateCollectionJobById = async ({ id, values = {}, collectionJobAttributes = COLLECTION_JOB_ATTRIBUTES, + collectionJobTestStatusAttributes = COLLECTION_JOB_TEST_STATUS_ATTRIBUTES, testPlanReportAttributes = TEST_PLAN_REPORT_ATTRIBUTES, testPlanRunAttributes = TEST_PLAN_RUN_ATTRIBUTES, testPlanVersionAttributes = TEST_PLAN_VERSION_ATTRIBUTES, @@ -415,6 +459,9 @@ const updateCollectionJobById = async ({ id, attributes: collectionJobAttributes, include: [ + collectionJobTestStatusAssociation( + collectionJobTestStatusAttributes + ), testPlanRunAssociation( testPlanRunAttributes, userAttributes, @@ -592,6 +639,30 @@ const restartCollectionJob = async ({ id }, { transaction }) => { return triggerWorkflow(job, [], { transaction }); }; +/** + * CollectionJobTestStatus updates + */ + +/** + * Update CollectionJobTestStatus entries in bulk via query. + * @param {object} options + * @param {object} options.where - values of the CollectionJobTestStatus record to be updated + * @param {object} options.values - values to be used to update columns for the record being referenced + * @param {*} options.transaction - Sequelize transaction + * @returns {Promise<*>} + */ +const updateCollectionJobTestStatusByQuery = ({ + where, + values = {}, + transaction +}) => { + return ModelService.update(CollectionJobTestStatus, { + values, + where, + transaction + }); +}; + module.exports = { // Basic CRUD createCollectionJob, @@ -603,5 +674,7 @@ module.exports = { scheduleCollectionJob, restartCollectionJob, cancelCollectionJob, - retryCanceledCollections + retryCanceledCollections, + // Basic CRUD for CollectionJobTestStatus + updateCollectionJobTestStatusByQuery }; diff --git a/server/models/services/helpers.js b/server/models/services/helpers.js index 32c75b7cb..f7f188c71 100644 --- a/server/models/services/helpers.js +++ b/server/models/services/helpers.js @@ -11,7 +11,8 @@ const { User, UserRoles, UserAts, - CollectionJob + CollectionJob, + CollectionJobTestStatus } = require('../index'); /** @@ -39,5 +40,8 @@ module.exports = { USER_ATTRIBUTES: getSequelizeModelAttributes(User), USER_ROLES_ATTRIBUTES: getSequelizeModelAttributes(UserRoles), USER_ATS_ATTRIBUTES: getSequelizeModelAttributes(UserAts), - COLLECTION_JOB_ATTRIBUTES: getSequelizeModelAttributes(CollectionJob) + COLLECTION_JOB_ATTRIBUTES: getSequelizeModelAttributes(CollectionJob), + COLLECTION_JOB_TEST_STATUS_ATTRIBUTES: getSequelizeModelAttributes( + CollectionJobTestStatus + ) }; diff --git a/server/resolvers/CollectionJob/index.js b/server/resolvers/CollectionJob/index.js new file mode 100644 index 000000000..64347cf91 --- /dev/null +++ b/server/resolvers/CollectionJob/index.js @@ -0,0 +1,3 @@ +const testStatus = require('./testStatusResolver'); +const testPlanRun = require('./testPlanRunResolver'); +module.exports = { testStatus, testPlanRun }; diff --git a/server/resolvers/CollectionJob/testPlanRunResolver.js b/server/resolvers/CollectionJob/testPlanRunResolver.js new file mode 100644 index 000000000..62014f8bd --- /dev/null +++ b/server/resolvers/CollectionJob/testPlanRunResolver.js @@ -0,0 +1,10 @@ +// eslint-disable-next-line no-unused-vars +const testPlanRunResolver = (collectionJob, _, context) => { + if (collectionJob.__testPlanRunChild) { + throw new Error( + 'can not request TestPlanRun.collectionJob.testPlanRun' + ); + } + return collectionJob.testPlanRun; +}; +module.exports = testPlanRunResolver; diff --git a/server/resolvers/CollectionJob/testStatusResolver.js b/server/resolvers/CollectionJob/testStatusResolver.js new file mode 100644 index 000000000..1c46a5d2b --- /dev/null +++ b/server/resolvers/CollectionJob/testStatusResolver.js @@ -0,0 +1,15 @@ +// eslint-disable-next-line no-unused-vars +const testStatusResolver = (collectionJob, _, context) => { + // resolve testStatus.test for each test + // testPlanRun "might" be null if the testPlanRun had been deleted + const testPlanTests = + collectionJob.testPlanRun?.testPlanReport.testPlanVersion.tests ?? []; + const tests = new Map(testPlanTests.map(test => [test.id, test])); + + return collectionJob.testStatus.map(status => ({ + ...status.dataValues, + // if not found, at least return the test id + test: tests.get(status.testId) ?? { id: status.testId } + })); +}; +module.exports = testStatusResolver; diff --git a/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js b/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js index ea53c36e9..c7240ce50 100644 --- a/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js +++ b/server/resolvers/CollectionJobOperations/cancelCollectionJobResolver.js @@ -2,7 +2,8 @@ const { AuthenticationError } = require('apollo-server'); const { updateCollectionJobById, - getCollectionJobById + getCollectionJobById, + updateCollectionJobTestStatusByQuery } = require('../../models/services/CollectionJobService'); const { COLLECTION_JOB_STATUS } = require('../../util/enums'); @@ -36,6 +37,11 @@ const cancelCollectionJobResolver = async ( if (collectionJob.status === COLLECTION_JOB_STATUS.COMPLETED) { return collectionJob; } else { + await updateCollectionJobTestStatusByQuery({ + where: { collectionJobId, status: COLLECTION_JOB_STATUS.QUEUED }, + values: { status: COLLECTION_JOB_STATUS.CANCELLED }, + transaction + }); return updateCollectionJobById({ id: collectionJobId, values: { status: COLLECTION_JOB_STATUS.CANCELLED }, diff --git a/server/resolvers/TestPlanRun/collectionJobResolver.js b/server/resolvers/TestPlanRun/collectionJobResolver.js new file mode 100644 index 000000000..ecfb6552c --- /dev/null +++ b/server/resolvers/TestPlanRun/collectionJobResolver.js @@ -0,0 +1,19 @@ +const collectionJobByTestPlanRunIdResolver = require('../collectionJobByTestPlanRunIdResolver'); + +const collectionJobResolver = async ( + testPlanRun, + args, // eslint-disable-line no-unused-vars + context // eslint-disable-line no-unused-vars +) => { + const collectionJob = await collectionJobByTestPlanRunIdResolver( + null, + { testPlanRunId: testPlanRun.id }, + context + ); + if (collectionJob) { + return { ...collectionJob.dataValues, __testPlanRunChild: true }; + } + return collectionJob; +}; + +module.exports = collectionJobResolver; diff --git a/server/resolvers/TestPlanRun/index.js b/server/resolvers/TestPlanRun/index.js index 74770a793..f542d26d4 100644 --- a/server/resolvers/TestPlanRun/index.js +++ b/server/resolvers/TestPlanRun/index.js @@ -1,7 +1,8 @@ const testResults = require('./testResultsResolver'); const testResultsLength = require('./testResultsLengthResolver'); - +const collectionJob = require('./collectionJobResolver'); module.exports = { + collectionJob, testResults, testResultsLength }; diff --git a/server/resolvers/index.js b/server/resolvers/index.js index 5e097b81d..5cf7dece6 100644 --- a/server/resolvers/index.js +++ b/server/resolvers/index.js @@ -41,6 +41,7 @@ const TestPlanRunOperations = require('./TestPlanRunOperations'); const TestResultOperations = require('./TestResultOperations'); const TestPlanVersionOperations = require('./TestPlanVersionOperations'); const CollectionJobOperations = require('./CollectionJobOperations'); +const CollectionJob = require('./CollectionJob'); const TestPlanRun = require('./TestPlanRun'); const Test = require('./Test'); const ScenarioResult = require('./ScenarioResult'); @@ -84,6 +85,7 @@ const resolvers = { AtOperations, AtVersionOperations, BrowserOperations, + CollectionJob, User, TestPlan, TestPlanVersion, diff --git a/server/routes/automation.js b/server/routes/automation.js index 460516af7..5d6540e64 100644 --- a/server/routes/automation.js +++ b/server/routes/automation.js @@ -10,9 +10,12 @@ const { handleError } = require('../middleware/handleError'); const router = Router(); -router.post('/:jobID/update', verifyAutomationScheduler, updateJobStatus); - -router.post('/:jobID/result', verifyAutomationScheduler, updateJobResults); +router.post('/:jobID', verifyAutomationScheduler, updateJobStatus); +router.post( + '/:jobID/test/:testRowNumber', + verifyAutomationScheduler, + updateJobResults +); router.use(handleError); diff --git a/server/services/GithubWorkflowService.js b/server/services/GithubWorkflowService.js index 9d21765d7..2fe571563 100644 --- a/server/services/GithubWorkflowService.js +++ b/server/services/GithubWorkflowService.js @@ -128,8 +128,8 @@ const createGithubWorkflow = async ({ job, directory, gitSha }) => { ); const browser = job.testPlanRun.testPlanReport.browser.name.toLowerCase(); const inputs = { - callback_url: `https://${callbackUrlHostname}/api/jobs/${job.id}/result`, - status_url: `https://${callbackUrlHostname}/api/jobs/${job.id}/update`, + callback_url: `https://${callbackUrlHostname}/api/jobs/${job.id}/test/:testRowNumber`, + status_url: `https://${callbackUrlHostname}/api/jobs/${job.id}`, callback_header: `x-automation-secret:${process.env.AUTOMATION_SCHEDULER_SECRET}`, work_dir: `tests/${directory}`, test_pattern: '{reference/**,test-*-nvda.*}', diff --git a/server/tests/integration/automation-scheduler.test.js b/server/tests/integration/automation-scheduler.test.js index 09f5b209b..0dcd800da 100644 --- a/server/tests/integration/automation-scheduler.test.js +++ b/server/tests/integration/automation-scheduler.test.js @@ -12,6 +12,7 @@ const markAsFinalResolver = require('../../resolvers/TestPlanReportOperations/ma const AtLoader = require('../../models/loaders/AtLoader'); const BrowserLoader = require('../../models/loaders/BrowserLoader'); const getGraphQLContext = require('../../graphql-context'); +const { COLLECTION_JOB_STATUS } = require('../../util/enums'); let mockAutomationSchedulerServer; let apiServer; @@ -138,6 +139,10 @@ const getTestCollectionJob = async (jobId, { transaction }) => } } } + testStatus { + test { id } + status + } } } `, @@ -166,6 +171,10 @@ const scheduleCollectionJobByMutation = async ({ transaction }) => } } } + testStatus { + test { id } + status + } } } `, @@ -288,7 +297,7 @@ describe('Automation controller', () => { }); }); - it('should cancel a job', async () => { + it('should cancel a job and all remaining tests', async () => { await dbCleaner(async transaction => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); @@ -300,6 +309,9 @@ describe('Automation controller', () => { const { collectionJob: storedCollectionJob } = await getTestCollectionJob(job.id, { transaction }); expect(storedCollectionJob.status).toEqual('CANCELLED'); + for (const test of storedCollectionJob.testStatus) { + expect(test.status).toEqual('CANCELLED'); + } }); }); @@ -342,9 +354,7 @@ describe('Automation controller', () => { await dbCleaner(async transaction => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); - const response = await sessionAgent.post( - `/api/jobs/${job.id}/update` - ); + const response = await sessionAgent.post(`/api/jobs/${job.id}`); expect(response.statusCode).toBe(403); expect(response.body).toEqual({ error: 'Unauthorized' @@ -357,7 +367,7 @@ describe('Automation controller', () => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); const response = await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'INVALID' }) .set( 'x-automation-secret', @@ -372,7 +382,7 @@ describe('Automation controller', () => { it('should fail to update a job status for a non-existent jobId', async () => { const response = await sessionAgent - .post(`/api/jobs/${444}/update`) + .post(`/api/jobs/${444}`) .send({ status: 'RUNNING' }) .set( 'x-automation-secret', @@ -389,7 +399,7 @@ describe('Automation controller', () => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); const response = await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'RUNNING' }) .set( 'x-automation-secret', @@ -421,7 +431,7 @@ describe('Automation controller', () => { const { scheduleCollectionJob: job } = await scheduleCollectionJobByMutation({ transaction }); const response = await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'CANCELLED', externalLogsUrl: 'https://www.aol.com/' @@ -462,7 +472,7 @@ describe('Automation controller', () => { transaction }); await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'RUNNING' }) .set( 'x-automation-secret', @@ -492,9 +502,8 @@ describe('Automation controller', () => { scenario => scenario.atId === at.id ).length; const response = await sessionAgent - .post(`/api/jobs/${job.id}/result`) + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) .send({ - testCsvRow: selectedTestRowNumber, capabilities: { atName: at.name, atVersion: at.atVersions[0].name, @@ -543,6 +552,118 @@ describe('Automation controller', () => { ); }); }); + // also marks status for test as COMPLETED + const { collectionJob: storedCollectionJob } = + await getTestCollectionJob(job.id, { transaction }); + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('RUNNING'); + const testStatus = storedCollectionJob.testStatus.find( + status => status.test.id === selectedTest.id + ); + expect(testStatus.status).toEqual(COLLECTION_JOB_STATUS.COMPLETED); + }); + }); + + it('should properly handle per-test status updates without capabilities present', async () => { + await apiServer.sessionAgentDbCleaner(async transaction => { + const { scheduleCollectionJob: job } = + await scheduleCollectionJobByMutation({ transaction }); + const collectionJob = await getCollectionJobById({ + id: job.id, + transaction + }); + // flag overall job as RUNNING + const externalLogsUrl = 'https://example.com/test/log/url'; + await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ status: 'RUNNING', externalLogsUrl }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + + const { tests } = + collectionJob.testPlanRun.testPlanReport.testPlanVersion; + const selectedTestIndex = 0; + const selectedTestRowNumber = 1; + const selectedTest = tests[selectedTestIndex]; + let response = await sessionAgent + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) + .send({ + status: COLLECTION_JOB_STATUS.RUNNING + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + let { collectionJob: storedCollectionJob } = + await getTestCollectionJob(job.id, { transaction }); + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('RUNNING'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + let foundStatus = false; + for (const testStatus of storedCollectionJob.testStatus) { + let expectedStatus = COLLECTION_JOB_STATUS.QUEUED; + if (testStatus.test.id === selectedTest.id) { + foundStatus = true; + expectedStatus = COLLECTION_JOB_STATUS.RUNNING; + } + expect(testStatus.status).toEqual(expectedStatus); + } + expect(foundStatus).toEqual(true); + + // check that putting this test into ERROR and sending an overall + // collection job ERROR will properly update things + response = await sessionAgent + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) + .send({ + status: COLLECTION_JOB_STATUS.ERROR + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + response = await sessionAgent + .post(`/api/jobs/${job.id}`) + .send({ + // avoiding sending externalLogsUrl here to test that when + // missing it is not overwritten/emptied. + status: COLLECTION_JOB_STATUS.ERROR + }) + .set( + 'x-automation-secret', + process.env.AUTOMATION_SCHEDULER_SECRET + ) + .set('x-transaction-id', transaction.id); + expect(response.statusCode).toBe(200); + + storedCollectionJob = ( + await getTestCollectionJob(job.id, { transaction }) + ).collectionJob; + expect(storedCollectionJob.id).toEqual(job.id); + expect(storedCollectionJob.status).toEqual('ERROR'); + expect(storedCollectionJob.externalLogsUrl).toEqual( + externalLogsUrl + ); + foundStatus = false; + for (const testStatus of storedCollectionJob.testStatus) { + let expectedStatus = COLLECTION_JOB_STATUS.CANCELLED; + if (testStatus.test.id === selectedTest.id) { + foundStatus = true; + expectedStatus = COLLECTION_JOB_STATUS.ERROR; + } + expect(testStatus.status).toEqual(expectedStatus); + } + expect(foundStatus).toEqual(true); }); }); @@ -588,7 +709,7 @@ describe('Automation controller', () => { transaction }); await sessionAgent - .post(`/api/jobs/${job.id}/update`) + .post(`/api/jobs/${job.id}`) .send({ status: 'RUNNING' }) .set( 'x-automation-secret', @@ -597,9 +718,8 @@ describe('Automation controller', () => { .set('x-transaction-id', transaction.id); const response = await sessionAgent - .post(`/api/jobs/${job.id}/result`) + .post(`/api/jobs/${job.id}/test/${selectedTestRowNumber}`) .send({ - testCsvRow: selectedTestRowNumber, capabilities: { atName: at.name, atVersion: atVersion.name, diff --git a/server/tests/integration/graphql.test.js b/server/tests/integration/graphql.test.js index 9bfeca0d8..20f8a4bf6 100644 --- a/server/tests/integration/graphql.test.js +++ b/server/tests/integration/graphql.test.js @@ -140,7 +140,8 @@ describe('graphql', () => { // 'TestResult' 'Issue', 'Vendor', - 'scheduleCollectionJob' + 'scheduleCollectionJob', + 'CollectionJobTestStatus' ]; const excludedTypeNameAndField = [ // Items formatted like this: @@ -159,6 +160,7 @@ describe('graphql', () => { ['Command', 'atOperatingMode'], // TODO: Include when v2 test format CI tests are done ['CollectionJob', 'testPlanRun'], ['CollectionJob', 'externalLogsUrl'], + ['CollectionJob', 'testStatus'], // These interact with Response Scheduler API // which is mocked in other tests. ['Mutation', 'scheduleCollectionJob'], @@ -504,10 +506,11 @@ describe('graphql', () => { releasedAt } } - testPlanRun(id: 3) { + testPlanRun(id: 1) { __typename id initiatedByAutomation + collectionJob { id } testPlanReport { id } @@ -781,6 +784,31 @@ describe('graphql', () => { ); }); + // esure recursive query of collectionJob<>testPlanRun fails at some depth + await expect( + typeAwareQuery( + gql` + query { + collectionJob(id: 1) { + id + testPlanRun { + id + collectionJob { + id + testPlanRun { + id + } + } + } + } + } + `, + { + transaction: false + } + ) + ).rejects.toBeDefined(); + expect(() => { const missingTypes = checkForMissingTypes(); if (missingTypes.length) { diff --git a/server/tests/util/mock-automation-scheduler-server.js b/server/tests/util/mock-automation-scheduler-server.js index eeec2dd23..ca9c65ec1 100644 --- a/server/tests/util/mock-automation-scheduler-server.js +++ b/server/tests/util/mock-automation-scheduler-server.js @@ -12,6 +12,9 @@ const { } = require('../../middleware/transactionMiddleware'); const { query } = require('../util/graphql-test-utilities'); +// 0 = no chance of test errors, 1 = always errors +const TEST_ERROR_CHANCE = 0; + const setupMockAutomationSchedulerServer = async () => { const app = express(); app.use(express.json()); @@ -26,9 +29,22 @@ const setupMockAutomationSchedulerServer = async () => { shutdownManager = new GracefulShutdownManager(listener); }); + const timeout = ms => + new Promise(resolve => setTimeout(() => resolve(), ms)); + const simulateJobStatusUpdate = async (jobId, newStatus) => { await axios.post( - `${process.env.APP_SERVER}/api/jobs/${jobId}/update`, + `${process.env.APP_SERVER}/api/jobs/${jobId}`, + { + status: newStatus + }, + axiosConfig + ); + }; + + const simulateTestStatusUpdate = async (jobId, testId, newStatus) => { + await axios.post( + `${process.env.APP_SERVER}/api/jobs/${jobId}/test/${testId}`, { status: newStatus }, @@ -66,22 +82,35 @@ const setupMockAutomationSchedulerServer = async () => { responses }; - testResult[isV2 ? 'presentationNumber' : 'testCsvRow'] = - currentTest.rowNumber; try { - await axios.post( - `${process.env.APP_SERVER}/api/jobs/${jobId}/result`, - testResult, - axiosConfig + await simulateTestStatusUpdate( + jobId, + currentTest.rowNumber, + COLLECTION_JOB_STATUS.RUNNING ); - } catch (e) { - // Likely just means the test was cancelled - return; - } + await timeout(Math.random() * 2000); + + if (Math.random() < TEST_ERROR_CHANCE) { + await simulateTestStatusUpdate( + jobId, + currentTest.rowNumber, + COLLECTION_JOB_STATUS.ERROR + ); + return simulateJobStatusUpdate( + jobId, + COLLECTION_JOB_STATUS.ERROR + ); + } else { + await axios.post( + `${process.env.APP_SERVER}/api/jobs/${jobId}/test/${currentTest.rowNumber}`, + testResult, + axiosConfig + ); + } - if (currentTestIndex < tests.length - 1) { - setTimeout(() => { - simulateResultCompletion( + if (currentTestIndex < tests.length - 1) { + await timeout(Math.random() * 5000); + return simulateResultCompletion( tests, atName, atVersionName, @@ -91,19 +120,12 @@ const setupMockAutomationSchedulerServer = async () => { currentTestIndex + 1, isV2 ); - }, Math.random() * 5000); - } else { - setTimeout( - () => - simulateJobStatusUpdate( - jobId, - COLLECTION_JOB_STATUS.COMPLETED - ), - 1000 - ); + } else { + simulateJobStatusUpdate(jobId, COLLECTION_JOB_STATUS.COMPLETED); + } + } catch (error) { + console.error('Error simulating collection job', error); } - - return testResult; }; app.post('/jobs/new', async (req, res) => { diff --git a/server/util/enums.js b/server/util/enums.js index ce07982d2..56ab4c579 100644 --- a/server/util/enums.js +++ b/server/util/enums.js @@ -6,6 +6,12 @@ const COLLECTION_JOB_STATUS = { CANCELLED: 'CANCELLED' }; +const isJobStatusFinal = status => + status === COLLECTION_JOB_STATUS.COMPLETED || + status === COLLECTION_JOB_STATUS.CANCELLED || + status === COLLECTION_JOB_STATUS.ERROR; + module.exports = { - COLLECTION_JOB_STATUS + COLLECTION_JOB_STATUS, + isJobStatusFinal };