From 79a0fc68b7dff4333a5b8d7b61a3dc6fe2a158e2 Mon Sep 17 00:00:00 2001
From: rossilor95 <rossilor95@outlook.it>
Date: Fri, 26 Jan 2024 21:49:38 +0100
Subject: [PATCH 1/4] test(wpt): refactor test statistics in WPTRunner class

---
 test/wpt/runner/runner.mjs | 76 +++++++++++++++++++++++---------------
 1 file changed, 46 insertions(+), 30 deletions(-)

diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs
index 8af9eb7c68d..c510a432106 100644
--- a/test/wpt/runner/runner.mjs
+++ b/test/wpt/runner/runner.mjs
@@ -58,11 +58,13 @@ export class WPTRunner extends EventEmitter {
   #reportPath
 
   #stats = {
-    completed: 0,
-    failed: 0,
-    success: 0,
+    subtestsCompleted: 0,
+    subtestsFailed: 0,
+    subtestsPassed: 0,
     expectedFailures: 0,
-    skipped: 0
+    testsFailed: 0,
+    testsPassed: 0,
+    testsSkipped: 0
   }
 
   constructor (folder, url, { appendReport = false, reportPath } = {}) {
@@ -158,7 +160,7 @@ export class WPTRunner extends EventEmitter {
       const status = resolveStatusPath(test, this.#status)
 
       if (status.file.skip || status.topLevel.skip) {
-        this.#stats.skipped += 1
+        this.#stats.testsSkipped += 1
 
         console.log(colors(`[${finishedFiles}/${total}] SKIPPED - ${test}`, 'yellow'))
         console.log('='.repeat(96))
@@ -187,19 +189,19 @@ export class WPTRunner extends EventEmitter {
           }
         })
 
-        let result, report
+        const fileUrl = new URL(`/${this.#folderName}${test.slice(this.#folderPath.length)}`, 'http://wpt')
+        fileUrl.pathname = fileUrl.pathname.replace(/\.js$/, '.html')
+        fileUrl.search = variant
+        const result = {
+          test: fileUrl.href.slice(fileUrl.origin.length),
+          subtests: [],
+          status: ''
+        }
+
+        let report
         if (this.#appendReport) {
           report = JSON.parse(readFileSync(this.#reportPath))
-
-          const fileUrl = new URL(`/${this.#folderName}${test.slice(this.#folderPath.length)}`, 'http://wpt')
-          fileUrl.pathname = fileUrl.pathname.replace(/\.js$/, '.html')
-          fileUrl.search = variant
-
-          result = {
-            test: fileUrl.href.slice(fileUrl.origin.length),
-            subtests: [],
-            status: 'OK'
-          }
+          result.status = 'OK'
           report.results.push(result)
         }
 
@@ -214,8 +216,8 @@ export class WPTRunner extends EventEmitter {
             this.handleTestCompletion(worker)
           } else if (message.type === 'error') {
             this.#uncaughtExceptions.push({ error: message.error, test })
-            this.#stats.failed += 1
-            this.#stats.success -= 1
+            this.#stats.subtestsFailed += 1
+            this.#stats.subtestsPassed -= 1
           }
         })
 
@@ -224,14 +226,21 @@ export class WPTRunner extends EventEmitter {
             signal: AbortSignal.timeout(timeout)
           })
 
-          console.log(colors(`[${finishedFiles}/${total}] PASSED - ${test}`, 'green'))
+          if (result?.subtests.every((subtest) => subtest.status === 'PASS')) {
+            this.#stats.testsPassed += 1
+            console.log(colors(`[${finishedFiles}/${total}] PASSED - ${test}`, 'green'))
+          } else {
+            this.#stats.testsFailed += 1
+            console.log(colors(`[${finishedFiles}/${total}] FAILED - ${test}`, 'red'))
+          }
+
           if (variant) console.log('Variant:', variant)
           console.log(`Test took ${(performance.now() - start).toFixed(2)}ms`)
           console.log('='.repeat(96))
         } catch (e) {
           console.log(`${test} timed out after ${timeout}ms`)
         } finally {
-          if (result?.subtests.length > 0) {
+          if (this.#appendReport && result?.subtests.length > 0) {
             writeFileSync(this.#reportPath, JSON.stringify(report))
           }
 
@@ -245,16 +254,16 @@ export class WPTRunner extends EventEmitter {
   }
 
   /**
-   * Called after a test has succeeded or failed.
+   * Called after a subtest has succeeded or failed.
    */
   handleIndividualTestCompletion (message, status, path, meta, wptResult) {
     const { file, topLevel } = status
 
     if (message.type === 'result') {
-      this.#stats.completed += 1
+      this.#stats.subtestsCompleted += 1
 
       if (message.result.status === 1) {
-        this.#stats.failed += 1
+        this.#stats.subtestsFailed += 1
 
         wptResult?.subtests.push({
           status: 'FAIL',
@@ -284,7 +293,7 @@ export class WPTRunner extends EventEmitter {
           status: 'PASS',
           name: sanitizeUnpairedSurrogates(message.result.name)
         })
-        this.#stats.success += 1
+        this.#stats.subtestsPassed += 1
       }
     }
   }
@@ -307,16 +316,23 @@ export class WPTRunner extends EventEmitter {
     }
 
     this.emit('completion')
-    const { completed, failed, success, expectedFailures, skipped } = this.#stats
+
+    const { testsPassed, testsFailed, testsSkipped } = this.#stats
+    console.log(
+      `Test    results for folder [${this.#folderName}]: ` +
+      `completed: ${this.#files.length}, passed: ${testsPassed}, failed: ${testsFailed}, ` +
+      `skipped: ${testsSkipped}`
+    )
+
+    const { subtestsCompleted, subtestsFailed, subtestsPassed, expectedFailures } = this.#stats
     console.log(
-      `[${this.#folderName}]: ` +
-      `completed: ${completed}, failed: ${failed}, success: ${success}, ` +
+      `Subtest results for folder [${this.#folderName}]: ` +
+      `completed: ${subtestsCompleted}, failed: ${subtestsFailed}, passed: ${subtestsPassed}, ` +
       `expected failures: ${expectedFailures}, ` +
-      `unexpected failures: ${failed - expectedFailures}, ` +
-      `skipped: ${skipped}`
+      `unexpected failures: ${subtestsFailed - expectedFailures}`
     )
 
-    process.exit(failed - expectedFailures ? 1 : process.exitCode)
+    process.exit(subtestsPassed - expectedFailures ? 1 : process.exitCode)
   }
 
   addInitScript (code) {

From ae2529adeefc5d0bd883538055bc925581077a51 Mon Sep 17 00:00:00 2001
From: rossilor95 <rossilor95@outlook.it>
Date: Fri, 26 Jan 2024 21:52:05 +0100
Subject: [PATCH 2/4] test(wpt): mark timed out tests as 'failed'

---
 test/wpt/runner/runner.mjs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs
index c510a432106..7cd1fa15f23 100644
--- a/test/wpt/runner/runner.mjs
+++ b/test/wpt/runner/runner.mjs
@@ -238,7 +238,13 @@ export class WPTRunner extends EventEmitter {
           console.log(`Test took ${(performance.now() - start).toFixed(2)}ms`)
           console.log('='.repeat(96))
         } catch (e) {
-          console.log(`${test} timed out after ${timeout}ms`)
+          // If the worker is terminated by the timeout signal, the test is marked as failed
+          this.#stats.testsFailed += 1
+          console.log(colors(`[${finishedFiles}/${total}] FAILED - ${test}`, 'red'))
+
+          if (variant) console.log('Variant:', variant)
+          console.log(`Test timed out after ${timeout}ms`)
+          console.log('='.repeat(96))
         } finally {
           if (this.#appendReport && result?.subtests.length > 0) {
             writeFileSync(this.#reportPath, JSON.stringify(report))

From ab088007f95d0cd7cccc01e955b040e9a275c1c6 Mon Sep 17 00:00:00 2001
From: rossilor95 <rossilor95@outlook.it>
Date: Tue, 30 Jan 2024 14:43:01 +0100
Subject: [PATCH 3/4] fixup

---
 test/wpt/runner/runner.mjs | 111 +++++++++++++++++++------------------
 1 file changed, 56 insertions(+), 55 deletions(-)

diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs
index 7cd1fa15f23..739f502a572 100644
--- a/test/wpt/runner/runner.mjs
+++ b/test/wpt/runner/runner.mjs
@@ -58,13 +58,13 @@ export class WPTRunner extends EventEmitter {
   #reportPath
 
   #stats = {
-    subtestsCompleted: 0,
-    subtestsFailed: 0,
-    subtestsPassed: 0,
+    completedTests: 0,
+    failedTests: 0,
+    passedTests: 0,
     expectedFailures: 0,
-    testsFailed: 0,
-    testsPassed: 0,
-    testsSkipped: 0
+    failedFiles: 0,
+    passedFiles: 0,
+    skippedFiles: 0
   }
 
   constructor (folder, url, { appendReport = false, reportPath } = {}) {
@@ -160,7 +160,7 @@ export class WPTRunner extends EventEmitter {
       const status = resolveStatusPath(test, this.#status)
 
       if (status.file.skip || status.topLevel.skip) {
-        this.#stats.testsSkipped += 1
+        this.#stats.skippedFiles += 1
 
         console.log(colors(`[${finishedFiles}/${total}] SKIPPED - ${test}`, 'yellow'))
         console.log('='.repeat(96))
@@ -216,8 +216,8 @@ export class WPTRunner extends EventEmitter {
             this.handleTestCompletion(worker)
           } else if (message.type === 'error') {
             this.#uncaughtExceptions.push({ error: message.error, test })
-            this.#stats.subtestsFailed += 1
-            this.#stats.subtestsPassed -= 1
+            this.#stats.failedTests += 1
+            this.#stats.passedTests -= 1
           }
         })
 
@@ -226,24 +226,24 @@ export class WPTRunner extends EventEmitter {
             signal: AbortSignal.timeout(timeout)
           })
 
-          if (result?.subtests.every((subtest) => subtest.status === 'PASS')) {
-            this.#stats.testsPassed += 1
-            console.log(colors(`[${finishedFiles}/${total}] PASSED - ${test}`, 'green'))
-          } else {
-            this.#stats.testsFailed += 1
+          if (result.subtests.some((subtest) => subtest?.isExpectedFailure === false)) {
+            this.#stats.failedFiles += 1
             console.log(colors(`[${finishedFiles}/${total}] FAILED - ${test}`, 'red'))
+          } else {
+            this.#stats.passedFiles += 1
+            console.log(colors(`[${finishedFiles}/${total}] PASSED - ${test}`, 'green'))
           }
 
           if (variant) console.log('Variant:', variant)
-          console.log(`Test took ${(performance.now() - start).toFixed(2)}ms`)
+          console.log(`Test File took ${(performance.now() - start).toFixed(2)}ms`)
           console.log('='.repeat(96))
         } catch (e) {
           // If the worker is terminated by the timeout signal, the test is marked as failed
-          this.#stats.testsFailed += 1
+          this.#stats.failedFiles += 1
           console.log(colors(`[${finishedFiles}/${total}] FAILED - ${test}`, 'red'))
 
           if (variant) console.log('Variant:', variant)
-          console.log(`Test timed out after ${timeout}ms`)
+          console.log(`Test File timed out after ${timeout}ms`)
           console.log('='.repeat(96))
         } finally {
           if (this.#appendReport && result?.subtests.length > 0) {
@@ -260,47 +260,48 @@ export class WPTRunner extends EventEmitter {
   }
 
   /**
-   * Called after a subtest has succeeded or failed.
+   * Called after a test has succeeded or failed.
    */
   handleIndividualTestCompletion (message, status, path, meta, wptResult) {
+    this.#stats.completedTests += 1
+
     const { file, topLevel } = status
+    const isFailure = message.result.status === 1
 
-    if (message.type === 'result') {
-      this.#stats.subtestsCompleted += 1
+    const testResult = {
+      status: isFailure? 'FAIL' : 'PASS',
+      name: sanitizeUnpairedSurrogates(message.result.name),
+    }
 
-      if (message.result.status === 1) {
-        this.#stats.subtestsFailed += 1
+    if (isFailure) {
+      this.#stats.failedTests += 1
 
-        wptResult?.subtests.push({
-          status: 'FAIL',
-          name: sanitizeUnpairedSurrogates(message.result.name),
-          message: sanitizeUnpairedSurrogates(message.result.message)
-        })
+      const name = normalizeName(message.result.name)
+      const sanitizedMessage = sanitizeUnpairedSurrogates(message.result.message)
 
-        const name = normalizeName(message.result.name)
+      if (file.flaky?.includes(name)) {
+        this.#stats.expectedFailures += 1
+        wptResult?.subtests.push({...testResult, message: sanitizedMessage, isExpectedFailure: true})
 
-        if (file.flaky?.includes(name)) {
-          this.#stats.expectedFailures += 1
-        } else if (file.allowUnexpectedFailures || topLevel.allowUnexpectedFailures || file.fail?.includes(name)) {
-          if (!file.allowUnexpectedFailures && !topLevel.allowUnexpectedFailures) {
-            if (Array.isArray(file.fail)) {
-              this.#statusOutput[path] ??= []
-              this.#statusOutput[path].push(name)
-            }
+      } else if (file.allowUnexpectedFailures || topLevel.allowUnexpectedFailures || file.fail?.includes(name)) {
+        if (!file.allowUnexpectedFailures && !topLevel.allowUnexpectedFailures) {
+          if (Array.isArray(file.fail)) {
+            this.#statusOutput[path] ??= []
+            this.#statusOutput[path].push(name)
           }
-
-          this.#stats.expectedFailures += 1
-        } else {
-          process.exitCode = 1
-          console.error(message.result)
         }
+        this.#stats.expectedFailures += 1
+        wptResult?.subtests.push({...testResult, message: sanitizedMessage, isExpectedFailure: true})
+
       } else {
-        wptResult?.subtests.push({
-          status: 'PASS',
-          name: sanitizeUnpairedSurrogates(message.result.name)
-        })
-        this.#stats.subtestsPassed += 1
+        wptResult?.subtests.push({...testResult, message: sanitizedMessage, isExpectedFailure: false})
+
+        process.exitCode = 1
+        console.error(message.result)
       }
+    } else {
+      this.#stats.passedTests += 1
+      wptResult?.subtests.push(testResult)
     }
   }
 
@@ -323,22 +324,22 @@ export class WPTRunner extends EventEmitter {
 
     this.emit('completion')
 
-    const { testsPassed, testsFailed, testsSkipped } = this.#stats
+    const { passedFiles, failedFiles, skippedFiles } = this.#stats
     console.log(
-      `Test    results for folder [${this.#folderName}]: ` +
-      `completed: ${this.#files.length}, passed: ${testsPassed}, failed: ${testsFailed}, ` +
-      `skipped: ${testsSkipped}`
+      `File results for folder [${this.#folderName}]: ` +
+      `completed: ${this.#files.length}, passed: ${passedFiles}, failed: ${failedFiles}, ` +
+      `skipped: ${skippedFiles}`
     )
 
-    const { subtestsCompleted, subtestsFailed, subtestsPassed, expectedFailures } = this.#stats
+    const { completedTests, failedTests, passedTests, expectedFailures } = this.#stats
     console.log(
-      `Subtest results for folder [${this.#folderName}]: ` +
-      `completed: ${subtestsCompleted}, failed: ${subtestsFailed}, passed: ${subtestsPassed}, ` +
+      `Test results for folder [${this.#folderName}]: ` +
+      `completed: ${completedTests}, failed: ${failedTests}, passed: ${passedTests}, ` +
       `expected failures: ${expectedFailures}, ` +
-      `unexpected failures: ${subtestsFailed - expectedFailures}`
+      `unexpected failures: ${failedTests - expectedFailures}`
     )
 
-    process.exit(subtestsPassed - expectedFailures ? 1 : process.exitCode)
+    process.exit(failedTests - expectedFailures ? 1 : process.exitCode)
   }
 
   addInitScript (code) {

From 6058d42f51e59ad7389241a928f572a6f89cf8a8 Mon Sep 17 00:00:00 2001
From: rossilor95 <rossilor95@outlook.it>
Date: Tue, 30 Jan 2024 20:23:42 +0100
Subject: [PATCH 4/4] fix: linting

---
 test/wpt/runner/runner.mjs | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/test/wpt/runner/runner.mjs b/test/wpt/runner/runner.mjs
index 739f502a572..117cffeb2fa 100644
--- a/test/wpt/runner/runner.mjs
+++ b/test/wpt/runner/runner.mjs
@@ -235,7 +235,7 @@ export class WPTRunner extends EventEmitter {
           }
 
           if (variant) console.log('Variant:', variant)
-          console.log(`Test File took ${(performance.now() - start).toFixed(2)}ms`)
+          console.log(`File took ${(performance.now() - start).toFixed(2)}ms`)
           console.log('='.repeat(96))
         } catch (e) {
           // If the worker is terminated by the timeout signal, the test is marked as failed
@@ -243,7 +243,7 @@ export class WPTRunner extends EventEmitter {
           console.log(colors(`[${finishedFiles}/${total}] FAILED - ${test}`, 'red'))
 
           if (variant) console.log('Variant:', variant)
-          console.log(`Test File timed out after ${timeout}ms`)
+          console.log(`File timed out after ${timeout}ms`)
           console.log('='.repeat(96))
         } finally {
           if (this.#appendReport && result?.subtests.length > 0) {
@@ -269,8 +269,8 @@ export class WPTRunner extends EventEmitter {
     const isFailure = message.result.status === 1
 
     const testResult = {
-      status: isFailure? 'FAIL' : 'PASS',
-      name: sanitizeUnpairedSurrogates(message.result.name),
+      status: isFailure ? 'FAIL' : 'PASS',
+      name: sanitizeUnpairedSurrogates(message.result.name)
     }
 
     if (isFailure) {
@@ -281,8 +281,7 @@ export class WPTRunner extends EventEmitter {
 
       if (file.flaky?.includes(name)) {
         this.#stats.expectedFailures += 1
-        wptResult?.subtests.push({...testResult, message: sanitizedMessage, isExpectedFailure: true})
-
+        wptResult?.subtests.push({ ...testResult, message: sanitizedMessage, isExpectedFailure: true })
       } else if (file.allowUnexpectedFailures || topLevel.allowUnexpectedFailures || file.fail?.includes(name)) {
         if (!file.allowUnexpectedFailures && !topLevel.allowUnexpectedFailures) {
           if (Array.isArray(file.fail)) {
@@ -291,11 +290,9 @@ export class WPTRunner extends EventEmitter {
           }
         }
         this.#stats.expectedFailures += 1
-        wptResult?.subtests.push({...testResult, message: sanitizedMessage, isExpectedFailure: true})
-
+        wptResult?.subtests.push({ ...testResult, message: sanitizedMessage, isExpectedFailure: true })
       } else {
-        wptResult?.subtests.push({...testResult, message: sanitizedMessage, isExpectedFailure: false})
-
+        wptResult?.subtests.push({ ...testResult, message: sanitizedMessage, isExpectedFailure: false })
         process.exitCode = 1
         console.error(message.result)
       }