[Merged by Bors] - Add ES5 and ES6 Conformance calculation to boa_tester

boa_tester/src/exec/mod.rs

@@ -3,8 +3,8 @@
 mod js262;
 
 use crate::{
-    read::ErrorType, Harness, Outcome, Phase, SuiteResult, Test, TestFlags, TestOutcomeResult,
-    TestResult, TestSuite,
+    read::ErrorType, Harness, Outcome, Phase, SpecVersion, Statistics, SuiteResult, Test,
+    TestFlags, TestOutcomeResult, TestResult, TestSuite,
 };
 use boa_engine::{
     context::ContextBuilder, job::SimpleJobQueue, native_function::NativeFunction,
@@ -57,53 +57,75 @@ impl TestSuite {
             println!();
         }
 
-        // Count passed tests
-        let mut passed = 0;
-        let mut ignored = 0;
-        let mut panic = 0;
+        // Count passed tests and es specs
+        let mut all = Statistics::default();
+        let mut es5 = Statistics::default();
+        let mut es6 = Statistics::default();
+
+        let mut append_stats = |spec_version: SpecVersion, f: &dyn Fn(&mut Statistics)| {
+            f(&mut all);
+            if spec_version == SpecVersion::ES5 {
+                f(&mut es5);
+            } else if spec_version == SpecVersion::ES6 {
+                f(&mut es6);
+            }
+        };
+
         for test in &tests {
             match test.result {
-                TestOutcomeResult::Passed => passed += 1,
-                TestOutcomeResult::Ignored => ignored += 1,
-                TestOutcomeResult::Panic => panic += 1,
+                TestOutcomeResult::Passed => {
+                    append_stats(test.spec_version, &|stats| {
+                        stats.passed += 1;
+                    });
+                }
+                TestOutcomeResult::Ignored => {
+                    append_stats(test.spec_version, &|stats| {
+                        stats.ignored += 1;
+                    });
+                }
+                TestOutcomeResult::Panic => {
+                    append_stats(test.spec_version, &|stats| {
+                        stats.panic += 1;
+                    });
+                }
                 TestOutcomeResult::Failed => {}
             }
+            append_stats(test.spec_version, &|stats| {
+                stats.total += 1;
+            });
         }
 
         // Count total tests
-        let mut total = tests.len();
         for suite in &suites {
-            total += suite.total;
-            passed += suite.passed;
-            ignored += suite.ignored;
-            panic += suite.panic;
+            all = all + suite.all_stats.clone();
+            es5 = es5 + suite.es5_stats.clone();
+            es6 = es6 + suite.es6_stats.clone();
             features.append(&mut suite.features.clone());
         }
 
         if verbose != 0 {
             println!(
-                "Suite {} results: total: {total}, passed: {}, ignored: {}, failed: {} (panics: \
+                "Suite {} results: total: {}, passed: {}, ignored: {}, failed: {} (panics: \
                     {}{}), conformance: {:.2}%",
+                all.total,
                 self.path.display(),
-                passed.to_string().green(),
-                ignored.to_string().yellow(),
-                (total - passed - ignored).to_string().red(),
-                if panic == 0 {
+                all.passed.to_string().green(),
+                all.ignored.to_string().yellow(),
+                (all.total - all.passed - all.ignored).to_string().red(),
+                if all.panic == 0 {
                     "0".normal()
                 } else {
-                    panic.to_string().red()
+                    all.panic.to_string().red()
                 },
-                if panic == 0 { "" } else { " ⚠" }.red(),
-                (passed as f64 / total as f64) * 100.0
+                if all.panic == 0 { "" } else { " ⚠" }.red(),
+                (all.passed as f64 / all.total as f64) * 100.0
             );
         }
-
         SuiteResult {
             name: self.name.clone(),
-            total,
-            passed,
-            ignored,
-            panic,
+            all_stats: all,
+            es5_stats: es5,
+            es6_stats: es6,
             suites,
             tests,
             features,
@@ -141,6 +163,7 @@ impl Test {
             }
             return TestResult {
                 name: self.name.clone(),
+                spec_version: self.spec_version,
                 strict,
                 result: TestOutcomeResult::Failed,
                 result_text: Box::from("Could not read test file.")
@@ -159,6 +182,7 @@ impl Test {
             }
             return TestResult {
                 name: self.name.clone(),
+                spec_version: self.spec_version,
                 strict,
                 result: TestOutcomeResult::Ignored,
                 result_text: Box::default(),
@@ -357,6 +381,7 @@ impl Test {
 
         TestResult {
             name: self.name.clone(),
+            spec_version: self.spec_version,
             strict,
             result,
             result_text: result_text.into_boxed_str(),

boa_tester/src/main.rs

@@ -90,6 +90,7 @@ use serde::{
 use std::{
     fs::{self, File},
     io::Read,
+    ops::Add,
     path::{Path, PathBuf},
 };
 
@@ -270,21 +271,32 @@ fn run_test_suite(
         }
         let results = suite.run(&harness, verbose, parallel);
 
+        let total = results.all_stats.total;
+        let passed = results.all_stats.passed;
+        let ignored = results.all_stats.ignored;
+        let panicked = results.all_stats.panic;
+
         println!();
         println!("Results:");
-        println!("Total tests: {}", results.total);
-        println!("Passed tests: {}", results.passed.to_string().green());
-        println!("Ignored tests: {}", results.ignored.to_string().yellow());
+        println!("Total tests: {total}");
+        println!("Passed tests: {}", passed.to_string().green());
+        println!("Ignored tests: {}", ignored.to_string().yellow());
         println!(
             "Failed tests: {} (panics: {})",
-            (results.total - results.passed - results.ignored)
-                .to_string()
-                .red(),
-            results.panic.to_string().red()
+            (total - passed - ignored).to_string().red(),
+            panicked.to_string().red()
         );
         println!(
             "Conformance: {:.2}%",
-            (results.passed as f64 / results.total as f64) * 100.0
+            (passed as f64 / total as f64) * 100.0
+        );
+        println!(
+            "ES5 Conformance: {:.2}%",
+            (results.es5_stats.passed as f64 / results.es5_stats.total as f64) * 100.0
+        );
+        println!(
+            "ES6 Conformance: {:.2}%",
+            (results.es6_stats.passed as f64 / results.es6_stats.total as f64) * 100.0
         );
 
         write_json(results, output, verbose)
@@ -318,19 +330,43 @@ struct TestSuite {
     tests: Box<[Test]>,
 }
 
-/// Outcome of a test suite.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct SuiteResult {
-    #[serde(rename = "n")]
-    name: Box<str>,
-    #[serde(rename = "c")]
+/// Represents a tests statistic
+#[derive(Default, Debug, Clone, Serialize, Deserialize)]
+struct Statistics {
+    #[serde(rename = "t")]
     total: usize,
     #[serde(rename = "o")]
     passed: usize,
     #[serde(rename = "i")]
     ignored: usize,
     #[serde(rename = "p")]
     panic: usize,
+}
+
+impl Add for Statistics {
+    type Output = Self;
+
+    fn add(self, rhs: Self) -> Self::Output {
+        Self {
+            total: self.total + rhs.total,
+            passed: self.passed + rhs.passed,
+            ignored: self.ignored + rhs.ignored,
+            panic: self.panic + rhs.panic,
+        }
+    }
+}
+
+/// Outcome of a test suite.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct SuiteResult {
+    #[serde(rename = "n")]
+    name: Box<str>,
+    #[serde(rename = "a")]
+    all_stats: Statistics,
+    #[serde(rename = "a5", default)]
+    es5_stats: Statistics,
+    #[serde(rename = "a6", default)]
+    es6_stats: Statistics,
     #[serde(skip_serializing_if = "Vec::is_empty", default)]
     #[serde(rename = "s")]
     suites: Vec<SuiteResult>,
@@ -348,6 +384,8 @@ struct SuiteResult {
 struct TestResult {
     #[serde(rename = "n")]
     name: Box<str>,
+    #[serde(rename = "v", default)]
+    spec_version: SpecVersion,
     #[serde(rename = "s", default)]
     strict: bool,
     #[serde(skip)]
@@ -368,13 +406,23 @@ enum TestOutcomeResult {
     Panic,
 }
 
+#[derive(Debug, Serialize, Clone, Copy, Deserialize, PartialEq, Default)]
+#[serde(untagged)]
+enum SpecVersion {
+    ES5 = 5,
+    ES6 = 6,
+    #[default]
+    ES13 = 13,
+}
+
 /// Represents a test.
 #[derive(Debug, Clone)]
 #[allow(dead_code)]
 struct Test {
     name: Box<str>,
     description: Box<str>,
     esid: Option<Box<str>>,
+    spec_version: SpecVersion,
     flags: TestFlags,
     information: Box<str>,
     features: Box<[Box<str>]>,
@@ -392,10 +440,19 @@ impl Test {
         N: Into<Box<str>>,
         C: Into<Box<Path>>,
     {
+        let spec_version = if metadata.es5id.is_some() {
+            SpecVersion::ES5
+        } else if metadata.es6id.is_some() {
+            SpecVersion::ES6
+        } else {
+            SpecVersion::ES13
+        };
+
         Self {
             name: name.into(),
             description: metadata.description,
             esid: metadata.esid,
+            spec_version,
             flags: metadata.flags.into(),
             information: metadata.info,
             features: metadata.features,

boa_tester/src/results.rs

@@ -1,3 +1,5 @@
+use crate::Statistics;
+
 use super::SuiteResult;
 use color_eyre::{eyre::WrapErr, Result};
 use serde::{Deserialize, Serialize};
@@ -25,14 +27,12 @@ struct ReducedResultInfo {
     commit: Box<str>,
     #[serde(rename = "u")]
     test262_commit: Box<str>,
-    #[serde(rename = "t")]
-    total: usize,
-    #[serde(rename = "o")]
-    passed: usize,
-    #[serde(rename = "i")]
-    ignored: usize,
-    #[serde(rename = "p")]
-    panic: usize,
+    #[serde(rename = "a")]
+    all_stats: Statistics,
+    #[serde(rename = "a5", default)]
+    es5_stats: Statistics,
+    #[serde(rename = "a6", default)]
+    es6_stats: Statistics,
 }
 
 impl From<ResultInfo> for ReducedResultInfo {
@@ -41,10 +41,9 @@ impl From<ResultInfo> for ReducedResultInfo {
         Self {
             commit: info.commit,
             test262_commit: info.test262_commit,
-            total: info.results.total,
-            passed: info.results.passed,
-            ignored: info.results.ignored,
-            panic: info.results.panic,
+            all_stats: info.results.all_stats,
+            es5_stats: info.results.es5_stats,
+            es6_stats: info.results.es6_stats,
         }
     }
 }
@@ -220,24 +219,24 @@ pub(crate) fn compare_results(base: &Path, new: &Path, markdown: bool) -> Result
     ))
     .wrap_err("could not read the new results")?;
 
-    let base_total = base_results.results.total as isize;
-    let new_total = new_results.results.total as isize;
+    let base_total = base_results.results.all_stats.total as isize;
+    let new_total = new_results.results.all_stats.total as isize;
     let total_diff = new_total - base_total;
 
-    let base_passed = base_results.results.passed as isize;
-    let new_passed = new_results.results.passed as isize;
+    let base_passed = base_results.results.all_stats.passed as isize;
+    let new_passed = new_results.results.all_stats.passed as isize;
     let passed_diff = new_passed - base_passed;
 
-    let base_ignored = base_results.results.ignored as isize;
-    let new_ignored = new_results.results.ignored as isize;
+    let base_ignored = base_results.results.all_stats.ignored as isize;
+    let new_ignored = new_results.results.all_stats.ignored as isize;
     let ignored_diff = new_ignored - base_ignored;
 
     let base_failed = base_total - base_passed - base_ignored;
     let new_failed = new_total - new_passed - new_ignored;
     let failed_diff = new_failed - base_failed;
 
-    let base_panics = base_results.results.panic as isize;
-    let new_panics = new_results.results.panic as isize;
+    let base_panics = base_results.results.all_stats.panic as isize;
+    let new_panics = new_results.results.all_stats.panic as isize;
     let panic_diff = new_panics - base_panics;
 
     let base_conformance = (base_passed as f64 / base_total as f64) * 100_f64;