feat(task): codegen test262 runtime test (#1959)

.github/workflows/conformance.yml

@@ -53,6 +53,12 @@ jobs:
       - name: Set up Bun
         uses: oven-sh/setup-bun@v1
 
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          registry-url: 'https://registry.npmjs.org'
+
       - run: cargo coverage
       - run: cargo run --release -p oxc_transform_conformance -- --exec
       - run: cargo run --release -p oxc_prettier_conformance

Cargo.toml

@@ -121,7 +121,7 @@ thiserror                 = { version = "1.0.56" }
 tokio                     = { version = "1" }
 tower-lsp                 = { version = "0.20.0", features = ["proposed"] }
 unicode-id-start          = { version = "1.1.2" }
-ureq                      = { version = "2.9.1", default-features = false, features = ["tls"] }
+ureq                      = { version = "2.9.1", default-features = false, features = ["tls", "json"] }
 url                       = { version = "2.5.0" }
 walkdir                   = { version = "2.4.0" }
 indexmap                  = { version = "2.1.0" }

tasks/coverage/Cargo.toml

@@ -40,6 +40,7 @@ pico-args    = { workspace = true }
 lazy_static  = { workspace = true }
 walkdir      = { workspace = true }
 regex        = { workspace = true }
+phf          = { workspace = true, features = ["macros"] }
 
 console        = "0.15.7"
 encoding_rs    = "0.8.33"

tasks/coverage/src/lib.rs

@@ -2,11 +2,15 @@ mod babel;
 mod codegen;
 mod minifier;
 mod misc;
+mod runtime;
 mod suite;
 mod test262;
 mod typescript;
 
-use std::path::PathBuf;
+use std::{path::PathBuf, process::Command};
+
+use runtime::CodegenRuntimeTest262Case;
+use similar::DiffableStr;
 
 use crate::{
     babel::{BabelCase, BabelSuite},
@@ -40,6 +44,7 @@ impl AppArgs {
     pub fn run_all(&self) {
         self.run_parser();
         self.run_codegen();
+        self.run_codegen_runtime();
         self.run_minifier();
     }
 
@@ -57,6 +62,24 @@ impl AppArgs {
         MiscSuite::<CodegenMiscCase>::new().run("codegen_misc", self);
     }
 
+    /// # Panics
+    pub fn run_codegen_runtime(&self) {
+        // Run runtime.js to test codegen runtime
+        let mut runtime_process = Command::new("node")
+            .args([
+                "--experimental-vm-modules",
+                project_root()
+                    .join("tasks/coverage/src/runtime/runtime.js")
+                    .to_string_lossy()
+                    .as_str()
+                    .unwrap_or_default(),
+            ])
+            .spawn()
+            .expect("Run runtime.js failed");
+        Test262Suite::<CodegenRuntimeTest262Case>::new().run("codegen_runtime_test262", self);
+        let _ = runtime_process.kill();
+    }
+
     pub fn run_minifier(&self) {
         Test262Suite::<MinifierTest262Case>::new().run("minifier_test262", self);
         BabelSuite::<MinifierBabelCase>::new().run("minifier_babel", self);

tasks/coverage/src/main.rs

@@ -24,6 +24,7 @@ fn main() {
     match task {
         "parser" => args.run_parser(),
         "codegen" => args.run_codegen(),
+        "codegen-runtime" => args.run_codegen_runtime(),
         "minifier" => args.run_minifier(),
         _ => args.run_all(),
     };

tasks/coverage/src/runtime/mod.rs

@@ -0,0 +1,161 @@
+use std::{
+    path::{Path, PathBuf},
+    time::Duration,
+};
+
+use oxc_tasks_common::{agent, project_root};
+use phf::{phf_set, Set};
+
+use oxc_allocator::Allocator;
+use oxc_codegen::{Codegen, CodegenOptions};
+use oxc_parser::Parser;
+use oxc_span::SourceType;
+use serde_json::json;
+
+use crate::{
+    suite::{Case, TestResult},
+    test262::{Test262Case, TestFlag},
+};
+
+static SKIP_EVALUATING_FEATURES: Set<&'static str> = phf_set! {
+  // Node's version of V8 doesn't implement these
+  "hashbang",
+  "legacy-regexp",
+  "regexp-duplicate-named-groups",
+  "symbols-as-weakmap-keys",
+  "tail-call-optimization",
+
+  // We don't care about API-related things
+  "ArrayBuffer",
+  "change-array-by-copy",
+  "DataView",
+  "resizable-arraybuffer",
+  "ShadowRealm",
+  "cross-realm",
+  "SharedArrayBuffer",
+  "String.prototype.toWellFormed",
+  "Symbol.match",
+  "Symbol.replace",
+  "Symbol.unscopables",
+  "Temporal",
+  "TypedArray",
+
+  // Added in oxc
+  "Array.fromAsync",
+  "IsHTMLDDA",
+  "iterator-helpers",
+  "set-methods",
+  "array-grouping",
+
+  // stage 2
+  "Intl.DurationFormat"
+};
+
+static SKIP_EVALUATING_THESE_INCLUDES: phf::Set<&'static str> = phf_set! {
+    // We don't preserve "toString()" on functions
+    "nativeFunctionMatcher.js",
+};
+
+const FIXTURES_PATH: &str = "tasks/coverage/test262/test";
+
+pub struct CodegenRuntimeTest262Case {
+    base: Test262Case,
+    test_root: PathBuf,
+}
+
+impl Case for CodegenRuntimeTest262Case {
+    fn new(path: PathBuf, code: String) -> Self {
+        Self { base: Test262Case::new(path, code), test_root: project_root().join(FIXTURES_PATH) }
+    }
+
+    fn code(&self) -> &str {
+        self.base.code()
+    }
+
+    fn path(&self) -> &Path {
+        self.base.path()
+    }
+
+    fn test_result(&self) -> &TestResult {
+        self.base.test_result()
+    }
+
+    fn skip_test_case(&self) -> bool {
+        self.base.should_fail()
+            || self
+                .base
+                .meta()
+                .includes
+                .iter()
+                .any(|include| SKIP_EVALUATING_THESE_INCLUDES.contains(include))
+            || self
+                .base
+                .meta()
+                .features
+                .iter()
+                .any(|feature| SKIP_EVALUATING_FEATURES.contains(feature))
+            || self.base.code().contains("$262")
+            || self.base.code().contains("$DONOTEVALUATE()")
+    }
+
+    fn run(&mut self) {
+        let result = {
+            let source_text = self.base.code();
+            let is_module = self.base.meta().flags.contains(&TestFlag::Module);
+            let is_only_strict = self.base.meta().flags.contains(&TestFlag::OnlyStrict);
+            let source_type = SourceType::default()
+                .with_module(is_module)
+                .with_always_strict(self.base.meta().flags.contains(&TestFlag::OnlyStrict));
+            let allocator = Allocator::default();
+            let program = Parser::new(&allocator, source_text, source_type).parse().program;
+            let mut codegen_source_text =
+                Codegen::<false>::new(source_text.len(), CodegenOptions).build(&program);
+            if is_only_strict {
+                codegen_source_text = format!("\"use strict\";\n{codegen_source_text}");
+            }
+            if is_module {
+                codegen_source_text = format!("{codegen_source_text}\n export {{}}");
+            }
+
+            self.run_test_code(codegen_source_text.as_str())
+        };
+        self.base.set_result(result);
+    }
+}
+
+impl CodegenRuntimeTest262Case {
+    fn run_test_code(&self, codegen_text: &str) -> TestResult {
+        let is_async = self.base.meta().flags.contains(&TestFlag::Async);
+        let is_module = self.base.meta().flags.contains(&TestFlag::Module);
+        let is_raw = self.base.meta().flags.contains(&TestFlag::Raw);
+        let import_dir = self
+            .test_root
+            .join(self.base.path().parent().map_or_else(|| unreachable!(), |p| p))
+            .to_string_lossy()
+            .to_string();
+        let result = agent()
+            .post("http://localhost:32055/run")
+            .timeout(Duration::from_secs(10))
+            .send_json(&json!({
+                "code": codegen_text,
+                "includes": self.base.meta().includes,
+                "isAsync": is_async,
+                "isModule": is_module,
+                "isRaw": is_raw,
+                "importDir": import_dir
+            }))
+            .map(|res| res.into_string().unwrap_or_default());
+
+        if let Err(error) = result {
+            TestResult::RuntimeError(error.to_string())
+        } else if let Ok(output) = result {
+            if output.is_empty() {
+                TestResult::Passed
+            } else {
+                TestResult::RuntimeError(output)
+            }
+        } else {
+            unreachable!()
+        }
+    }
+}

tasks/coverage/src/runtime/package.json

@@ -0,0 +1 @@
+{ "type": "module" }