Skip to content

Commit

Permalink
fail wpt runner if expected failures does not match actual (#3236)
Browse files Browse the repository at this point in the history
  • Loading branch information
KhafraDev authored May 10, 2024
1 parent 0a66d30 commit 3b1b2fb
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 26 deletions.
32 changes: 30 additions & 2 deletions test/wpt/runner/runner.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ export class WPTRunner extends EventEmitter {
if (message.type === 'result') {
this.handleIndividualTestCompletion(message, status, test, meta, result)
} else if (message.type === 'completion') {
this.handleTestCompletion(worker)
this.handleTestCompletion(worker, status, test)
} else if (message.type === 'error') {
this.#uncaughtExceptions.push({ error: message.error, test })
this.#stats.failedTests += 1
Expand Down Expand Up @@ -318,8 +318,36 @@ export class WPTRunner extends EventEmitter {
* Called after all the tests in a worker are completed.
* @param {Worker} worker
*/
handleTestCompletion (worker) {
handleTestCompletion (worker, status, path) {
worker.terminate()

const { file } = status
const hasExpectedFailures = !!file.fail
const testHasFailures = !!this.#statusOutput?.[path]
const failed = this.#statusOutput?.[path] ?? []

if (hasExpectedFailures !== testHasFailures) {
console.log({ expected: file.fail, failed })

if (failed.length === 0) {
console.log(colors('Tests are marked as failure but did not fail, yay!', 'red'))
} else if (!hasExpectedFailures) {
console.log(colors('Test failed but there were no expected errors.', 'red'))
}

process.exitCode = 1
} else if (hasExpectedFailures && testHasFailures) {
const diff = [
...file.fail.filter(x => !failed.includes(x)),
...failed.filter(x => !file.fail.includes(x))
]

if (diff.length) {
console.log({ diff })
console.log(colors('Expected failures did not match actual failures', 'red'))
process.exitCode = 1
}
}
}

/**
Expand Down
2 changes: 1 addition & 1 deletion test/wpt/runner/util.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -168,5 +168,5 @@ export function resolveStatusPath (path, status) {
}
}

return { topLevel: status ?? {}, file: status?.[paths.at(-1)] ?? {} }
return { fullPath: path, topLevel: status ?? {}, file: status?.[paths.at(-1)] ?? {} }
}
3 changes: 2 additions & 1 deletion test/wpt/status/FileAPI.status.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
"skip": true
},
"Blob-stream.any.js": {
"fail": [
"note": "fails in node v18",
"flaky": [
"Reading Blob.stream() with BYOB reader"
]
}
Expand Down
14 changes: 4 additions & 10 deletions test/wpt/status/fetch.status.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,7 @@
"abort": {
"general.any.js": {
"fail": [
"Already aborted signal rejects immediately",
"Underlying connection is closed when aborting after receiving response - no-cors",
"Stream errors once aborted. Underlying connection closed.",
"Readable stream synchronously cancels with AbortError if aborted before reading"
"Already aborted signal rejects immediately"
]
},
"cache.https.any.js": {
Expand Down Expand Up @@ -52,9 +49,6 @@
},
"referrer.any.js": {
"fail": [
"origin-when-cross-origin policy on a cross-origin URL",
"origin-when-cross-origin policy on a cross-origin URL after same-origin redirection",
"origin-when-cross-origin policy on a same-origin URL after cross-origin redirection",
"origin-when-cross-origin policy on a same-origin URL"
]
},
Expand Down Expand Up @@ -100,7 +94,7 @@
},
"stream-safe-creation.any.js": {
"note": "tests are very finnicky",
"fail": [
"flaky": [
"throwing Object.prototype.type accessor should not affect stream creation by 'fetch'",
"Object.prototype.type accessor returning invalid value should not affect stream creation by 'fetch'",
"throwing Object.prototype.highWaterMark accessor should not affect stream creation by 'fetch'",
Expand Down Expand Up @@ -344,7 +338,6 @@
"Adding invalid request header \"Accept-Encoding: KO\"",
"Adding invalid request header \"Access-Control-Request-Headers: KO\"",
"Adding invalid request header \"Access-Control-Request-Method: KO\"",
"Adding invalid request header \"Access-Control-Request-Private-Network: KO\"",
"Adding invalid request header \"Connection: KO\"",
"Adding invalid request header \"Content-Length: KO\"",
"Adding invalid request header \"Cookie: KO\"",
Expand Down Expand Up @@ -394,7 +387,8 @@
]
},
"response-consume-stream.any.js": {
"fail": [
"note": "only fail in node v18",
"flaky": [
"Read blob response's body as readableStream with mode=byob",
"Read text response's body as readableStream with mode=byob",
"Read URLSearchParams response's body as readableStream with mode=byob",
Expand Down
12 changes: 0 additions & 12 deletions test/wpt/status/websockets.status.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,6 @@
}
}
},
"Create-blocked-port.any.js": {
"note": "TODO(@KhafraDev): investigate failure",
"fail": [
"Basic check"
]
},
"Send-binary-arraybufferview-float32.any.js": {
"note": "TODO(@KhafraDev): investigate failure",
"fail": [
Expand Down Expand Up @@ -58,12 +52,6 @@
"Send binary data on a WebSocket - ArrayBufferView - Uint32Array with offset - Connection should be closed"
]
},
"basic-auth.any.js": {
"note": "TODO(@KhafraDev): investigate failure",
"fail": [
"HTTP basic authentication should work with WebSockets"
]
},
"Create-on-worker-shutdown.any.js": {
"skip": true,
"//": "Node.js workers are different from web workers & don't work with blob: urls"
Expand Down

0 comments on commit 3b1b2fb

Please sign in to comment.