Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Per-test status delivery #47

Merged
merged 8 commits into from
Apr 24, 2024
Merged

Per-test status delivery #47

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
94ee15b
per test status updates
gnarf Mar 21, 2024
8b00f61
snapshot diffs
gnarf Mar 21, 2024
eda1c25
grab the presNumber/testId from the info
gnarf Mar 22, 2024
db67ee2
updates from review
gnarf Apr 3, 2024
a9823e7
update to per-test "RUNNING" or "ERROR" status
gnarf Apr 3, 2024
591ce4c
update snapshots
gnarf Apr 3, 2024
60d7197
add in support for dynamic :testRowNumber in callback url
gnarf Apr 10, 2024
16a0650
missing snapshot update
gnarf Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 42 additions & 24 deletions src/host/main.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,23 @@ export async function hostMain({

const callbackRequests = [];

const postCallbackWhenEnabled = async body => {
// ignore if not in callback mode
if (!callbackUrl) return;
const headers = {
'Content-Type': 'application/json',
...(callbackHeader || {}),
};
await Promise.allSettled(callbackRequests);
gnarf marked this conversation as resolved.
Show resolved Hide resolved
callbackRequests.push(
fetch(callbackUrl, {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code appears to use the same URL for test status updates as it has already been using for AT responses ("results"). Is that right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wasn't sure what we were going to do for the API on the frontend about it. It would be most convenient if it did use the same URL (no need to pass a second URL to the automation harness), and it does make some sense as these messages are all about a specific test results collection, not the entirety of the job.

My first thoughts were to update the current "results" url to also look for error or status: "TEST_START" and update appropriately.

method: 'post',
body: JSON.stringify(body),
headers,
}).then(logUnsuccessfulHTTP.bind(null, log))
);
};

for (const test of plan.tests) {
log(HostMessage.START_TEST);
const testLogJob = startJob(async function (signal) {
Expand All @@ -78,41 +95,42 @@ export async function hostMain({
});

const file = plan.files.find(({ name }) => name === test.filepath);
const result = await agent.run(JSON.parse(textDecoder.decode(file.bufferData)));
if (callbackUrl) {
const headers = {
'Content-Type': 'application/json',
...(callbackHeader || {}),
};
const { testId, presentationNumber, capabilities, commands } = result;
const body = JSON.stringify({
testCsvRow: testId,
presentationNumber,
const testSource = JSON.parse(textDecoder.decode(file.bufferData));

const callbackBody = {
testCsvRow: testSource.info.testId,
presentationNumber: testSource.info.presentationNumber,
};

try {
postCallbackWhenEnabled({ ...callbackBody, status: 'RUNNING' });

const result = await agent.run(testSource);

const { capabilities, commands } = result;

postCallbackWhenEnabled({
...callbackBody,
capabilities,
responses: commands.map(({ output }) => output),
// a v2 of this API should allow the aria at app to parse the capabilities we are sending instead.
atVersionName: capabilities.atVersion,
browserVersionName: capabilities.browserVersion,
});
callbackRequests.push(
fetch(callbackUrl, {
method: 'post',
body,
headers,
}).then(logUnsuccessfulHTTP.bind(null, log))
);

plan = addTestResultToTestPlan(plan, test.filepath, result);
} catch (exception) {
const error = `${exception.message ?? exception}`;
log(HostMessage.TEST_ERROR, { error });
postCallbackWhenEnabled({ ...callbackBody, error, status: 'ERROR' });
} finally {
await testLogJob.cancel();
}
plan = addTestResultToTestPlan(plan, test.filepath, result);
await Promise.allSettled(callbackRequests);
await testLogJob.cancel();
}

server.removeFiles(serverDirectory);
log(HostMessage.REMOVE_SERVER_DIRECTORY, { url: serverDirectory.baseUrl });

log(HostMessage.STOP_AGENT);
await agent.stop();

await Promise.allSettled(callbackRequests);
await emitPlanResults(plan);
}

Expand Down
3 changes: 3 additions & 0 deletions src/host/messages.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ export const HostMessage = {
START_TEST: 'startTest',
/** @type {'reportingError'} */
REPORTING_ERROR: 'reportingError',
/** @type {'testError'} */
TEST_ERROR: 'testError',
};

export const HOST_TEMPLATES = {
Expand All @@ -63,6 +65,7 @@ export const HOST_TEMPLATES = {
[HostMessage.AGENT_LOG]: ({ text }) => `[Agent]: ${text}`,
[HostMessage.AGENT_CRASHED]: () => `Agent crashed.`,
[HostMessage.START_TEST]: () => `Starting test.`,
[HostMessage.TEST_ERROR]: ({ error }) => `Test Error ${error}`,
[HostMessage.REPORTING_ERROR]: ({ status, body }) =>
`HTTP ${status} response received when reporting result: '${body}'.`,
};
Expand Down
36 changes: 18 additions & 18 deletions src/host/tests/snapshots/agent.js.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -210,7 +210,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -278,7 +278,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -479,7 +479,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -506,7 +506,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -738,7 +738,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -765,7 +765,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -958,7 +958,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -985,7 +985,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -1300,7 +1300,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -1327,7 +1327,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)
> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)

{
capabilities: {
Expand All @@ -1354,7 +1354,7 @@ Generated by [AVA](https://avajs.dev).
testId: 2,
}

> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -1700,7 +1700,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -1727,7 +1727,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)
> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)

{
capabilities: {
Expand All @@ -1754,7 +1754,7 @@ Generated by [AVA](https://avajs.dev).
testId: 2,
}

> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -2072,7 +2072,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -2099,7 +2099,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)
> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)

{
capabilities: {
Expand All @@ -2126,7 +2126,7 @@ Generated by [AVA](https://avajs.dev).
testId: 2,
}

> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down
Binary file modified src/host/tests/snapshots/agent.js.snap
View file
Open in desktop
Binary file not shown.
Loading
Loading