Merge pull request #65 from eval-exec/feat/customize-union-id

[Feature] Add custom union ID syntax

Makefile

@@ -5,7 +5,7 @@ ci:
 	make ci-examples ci-crates; \
 	echo "Success!"
 
-RUST_DEV_PROJS = examples/ci-tests
+RUST_DEV_PROJS = examples/ci-tests tests
 RUST_PROD_PROJS = bindings/rust tools/codegen tools/compiler
 RUST_PROJS = ${RUST_DEV_PROJS} ${RUST_PROD_PROJS}
 C_PROJS = examples/ci-tests

tests/.gitignore

@@ -0,0 +1,2 @@
+target
+Cargo.lock

tests/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "tests"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+[dev-dependencies]
+codegen-0_7_3 = {package = "molecule-codegen", version = "0.7.3", features = ["compiler-plugin"]}
+codegen-dev = {package = "molecule-codegen", path = "../tools/codegen", features = ["compiler-plugin"]}
+molecule = "0.7.3"
+
+[build-dependencies]
+codegen-0_7_3 = {package = "molecule-codegen", version = "0.7.3", features = ["compiler-plugin"]}
+codegen-dev = {package = "molecule-codegen", path = "../tools/codegen", features = ["compiler-plugin"]}
+molecule = "0.7.3"

tests/build.rs

@@ -0,0 +1,66 @@
+fn compile_schema_0_7_3(schema: &str) {
+    let out_dir = std::path::PathBuf::from(&std::env::var("OUT_DIR").unwrap()).join("0_7_3");
+    std::fs::create_dir_all(&out_dir).unwrap();
+
+    let mut compiler = codegen_0_7_3::Compiler::new();
+    compiler
+        .input_schema_file(schema)
+        .generate_code(codegen_0_7_3::Language::Rust)
+        .output_dir(out_dir)
+        .run()
+        .unwrap();
+    println!("cargo:rerun-if-changed={}", schema);
+}
+
+fn compile_schema_dev(schema: &str) {
+    let out_dir = std::path::PathBuf::from(&std::env::var("OUT_DIR").unwrap()).join("dev");
+    std::fs::create_dir_all(&out_dir).unwrap();
+
+    let mut compiler = codegen_dev::Compiler::new();
+    compiler
+        .input_schema_file(schema)
+        .generate_code(codegen_dev::Language::Rust)
+        .output_dir(out_dir)
+        .run()
+        .unwrap();
+    println!("cargo:rerun-if-changed={}", schema);
+}
+
+fn compile_intermediate_0_7_3(schema: &str) {
+    let out_dir = std::path::PathBuf::from(&std::env::var("OUT_DIR").unwrap()).join("0_7_3");
+    std::fs::create_dir_all(&out_dir).unwrap();
+
+    let mut compiler = codegen_0_7_3::Compiler::new();
+    compiler
+        .input_schema_file(schema)
+        .generate_intermediate(codegen_0_7_3::IntermediateFormat::JSON)
+        .output_dir(out_dir)
+        .run()
+        .unwrap();
+    println!("cargo:rerun-if-changed={}", schema);
+}
+
+fn compile_intermediate_dev(schema: &str) {
+    let out_dir = std::path::PathBuf::from(&std::env::var("OUT_DIR").unwrap()).join("dev");
+    std::fs::create_dir_all(&out_dir).unwrap();
+
+    let mut compiler = codegen_dev::Compiler::new();
+    compiler
+        .input_schema_file(schema)
+        .generate_intermediate(codegen_dev::IntermediateFormat::JSON)
+        .output_dir(out_dir)
+        .run()
+        .unwrap();
+    println!("cargo:rerun-if-changed={}", schema);
+}
+
+fn main() {
+    println!("cargo:rerun-if-changed=./union_foo_0_7_3.mol");
+    println!("cargo:rerun-if-changed=./union_foo_with_custom_id.mol");
+
+    compile_intermediate_0_7_3("./union_foo_0_7_3.mol");
+    compile_intermediate_dev("./union_foo_with_custom_id.mol");
+
+    compile_schema_0_7_3("./union_foo_0_7_3.mol");
+    compile_schema_dev("./union_foo_with_custom_id.mol");
+}

tests/src/main.rs

@@ -0,0 +1,3 @@
+mod union_compatibility_test;
+
+fn main() {}

tests/src/union_compatibility_test.rs

@@ -0,0 +1,73 @@
+#[cfg(test)]
+mod tests {
+    use molecule::prelude::*;
+
+    static UNION_FOO_0_7_3_JSON_INTERMEDIATE: &str =
+        include_str!(concat!(env!("OUT_DIR"), "/0_7_3/union_foo_0_7_3.json"));
+
+    static UNION_FOO_DEV_JSON_INTERMEDIATE: &str = include_str!(concat!(
+        env!("OUT_DIR"),
+        "/dev/union_foo_with_custom_id.json"
+    ));
+
+    #[test]
+    fn test_recover_0_7_3_intermediate_by_current_ir_recover() {
+        let format = codegen_dev::IntermediateFormat::JSON;
+        let ast_result = format.recover(UNION_FOO_0_7_3_JSON_INTERMEDIATE.as_bytes());
+        assert!(ast_result.is_ok());
+    }
+
+    #[test]
+    fn test_recover_ir() {
+        let format = codegen_dev::IntermediateFormat::JSON;
+        let ast_result = format.recover(UNION_FOO_DEV_JSON_INTERMEDIATE.as_bytes());
+        assert!(ast_result.is_ok());
+    }
+
+    mod union_foo_0_7_3 {
+        #![allow(clippy::all, dead_code)]
+        include!(concat!(env!("OUT_DIR"), "/0_7_3/union_foo_0_7_3.rs"));
+    }
+
+    mod union_foo_dev {
+        #![allow(clippy::all, dead_code)]
+        include!(concat!(env!("OUT_DIR"), "/dev/union_foo_with_custom_id.rs"));
+    }
+
+    #[test]
+    fn test_decode_0_7_3_generated_rust_bytes_by_current_version() {
+        let a2_0_7_3 = union_foo_0_7_3::A2::new_builder()
+            .nth0(Byte::from(17))
+            .build();
+
+        let foo_0_7_3 = union_foo_0_7_3::Foo::new_builder()
+            .set(a2_0_7_3.clone())
+            .build();
+        let foo_0_7_3_slice = foo_0_7_3.as_slice();
+
+        let foo_dev_result = union_foo_dev::FooOnlyReserveA2AndA3::from_slice(foo_0_7_3_slice);
+        assert!(foo_dev_result.is_ok());
+        let foo_dev = foo_dev_result.unwrap();
+
+        let foo_union_dev = foo_dev.to_enum();
+
+        if let union_foo_dev::FooOnlyReserveA2AndA3Union::A2(a2_dev) = foo_union_dev {
+            assert_eq!(a2_0_7_3.as_slice(), a2_dev.as_slice());
+        } else {
+            panic!("foo_union_dev should be A2");
+        }
+    }
+
+    #[test]
+    fn test_decode_0_7_3_generated_deprecated_rust_bytes_by_current_version() {
+        let a0_0_7_3 = union_foo_0_7_3::A0::new_builder()
+            .nth0(Byte::from(133))
+            .build();
+
+        let foo_0_7_3 = union_foo_0_7_3::Foo::new_builder().set(a0_0_7_3).build();
+        let foo_0_7_3_slice = foo_0_7_3.as_slice();
+
+        let foo_dev_result = union_foo_dev::FooOnlyReserveA2AndA3::from_slice(foo_0_7_3_slice);
+        assert!(foo_dev_result.is_err());
+    }
+}

tests/union_foo_0_7_3.mol

@@ -0,0 +1,11 @@
+array a0 [byte;1];
+array a1 [byte;2];
+array a2 [byte;3];
+array a3 [byte;4];
+
+union Foo {
+  a0,
+  a1,
+  a2,
+  a3,
+}

tests/union_foo_with_custom_id.mol

@@ -0,0 +1,7 @@
+array a2 [byte;3];
+array a3 [byte;4];
+
+union Foo_Only_Reserve_a2_and_a3{
+  a2 : 2,
+  a3 : 3,
+}

tools/codegen/Cargo.toml

@@ -29,6 +29,9 @@ serde = { version = "1.0.118", features = ["derive", "rc"], optional = true }
 serde_json = { version = "1.0.61", optional = true }
 serde_yaml = { version = "0.8.15", optional = true }
 
+[dev-dependencies]
+tempfile = "3"
+
 [features]
 default = []
 compiler-plugin = ["serde", "serde_json", "serde_yaml"]

tools/codegen/src/ast/mod.rs

@@ -3,5 +3,5 @@ pub(crate) mod verified;
 
 pub use verified::{
     Array, Ast, DefaultContent, DynVec, FieldDecl, FixVec, HasName, ImportStmt, ItemDecl, Option_,
-    Primitive, Struct, Table, TopDecl, Union,
+    Primitive, Struct, Table, TopDecl, Union, UnionItemDecl,
 };

tools/codegen/src/ast/raw/mod.rs

@@ -40,7 +40,7 @@ pub(crate) struct OptionDecl {
 #[derive(Debug, Property)]
 pub(crate) struct UnionDecl {
     name: String,
-    items: Vec<ItemDecl>,
+    items: Vec<CustomUnionItemDecl>,
     imported_depth: usize,
 }
 
@@ -78,6 +78,12 @@ pub(crate) struct ItemDecl {
     typ: String,
 }
 
+#[derive(Debug, Property)]
+pub(crate) struct CustomUnionItemDecl {
+    typ: String,
+    id: usize,
+}
+
 #[derive(Debug, Property)]
 pub(crate) struct FieldDecl {
     name: String,

tools/codegen/src/ast/raw/utils.rs

@@ -1,8 +1,10 @@
+use std::collections::HashSet;
 use std::{ffi, fs, io::Read as _, path::Path, str::FromStr};
 
 use pest::{error::Error as PestError, iterators::Pairs, Parser as _};
 use same_file::is_same_file;
 
+use crate::ast::raw::CustomUnionItemDecl;
 use crate::{
     ast::raw as ast,
     parser,
@@ -40,6 +42,51 @@ impl<'i> utils::PairsUtils for Pairs<'i, parser::Rule> {
         ret
     }
 
+    fn next_custom_union_items(&mut self) -> Vec<CustomUnionItemDecl> {
+        let mut previous_id: Option<usize> = None;
+        let mut ret = Vec::new();
+
+        let mut custom_ids = HashSet::new();
+        for item in self {
+            match item.as_rule() {
+                parser::Rule::item_decl => {
+                    let mut pair = item.into_inner();
+                    let node = ast::CustomUnionItemDecl {
+                        typ: pair.next_string(),
+                        id: if let Some(pre_id) = previous_id {
+                            pre_id + 1
+                        } else {
+                            0
+                        },
+                    };
+                    pair.next_should_be_none();
+                    ret.push(node);
+                }
+                parser::Rule::custom_union_item_decl => {
+                    let mut pair = item.into_inner();
+                    let node = ast::CustomUnionItemDecl {
+                        typ: pair.next_string(),
+                        id: pair.next_usize(),
+                    };
+                    pair.next_should_be_none();
+                    ret.push(node);
+                }
+                _ => unreachable!(),
+            }
+
+            if !custom_ids.insert(ret.last().unwrap().id) {
+                panic!(
+                    "Custom Union Item ID {} is duplicated",
+                    ret.last().unwrap().id
+                );
+            }
+            previous_id = Some(ret.last().unwrap().id);
+        }
+        // union items should be sorted by custom ID
+        ret.sort_by_key(|item| item.id);
+        ret
+    }
+
     fn next_fields(&mut self) -> Vec<ast::FieldDecl> {
         let mut ret = Vec::new();
         for field in self {
@@ -204,7 +251,7 @@ impl parser::Parser {
                     let mut pair = pair.into_inner();
                     let node = ast::UnionDecl {
                         name: pair.next_string(),
-                        items: pair.next_items(),
+                        items: pair.next_custom_union_items(),
                         imported_depth,
                     };
                     pair.next_should_be_none();

tools/codegen/src/ast/verified/complete.rs

@@ -31,7 +31,10 @@ impl CompleteRawDecl for raw::UnionDecl {
         }
         self.items()
             .iter()
-            .map(|raw_item| deps.get(raw_item.typ()).map(super::ItemDecl::new))
+            .map(|raw_item| {
+                deps.get(raw_item.typ())
+                    .map(|typ| super::UnionItemDecl::new(typ, raw_item.id()))
+            })
             .collect::<Option<Vec<_>>>()
             .map(|items| {
                 let name = self.name().to_owned();

tools/codegen/src/ast/verified/mod.rs

@@ -61,7 +61,7 @@ pub struct Option_ {
 #[property(get(public))]
 pub struct Union {
     name: String,
-    items: Vec<ItemDecl>,
+    items: Vec<UnionItemDecl>,
     imported_depth: usize,
 }
 
@@ -115,6 +115,13 @@ pub struct ItemDecl {
     typ: Rc<TopDecl>,
 }
 
+#[derive(Debug, Property)]
+#[property(get(public))]
+pub struct UnionItemDecl {
+    typ: Rc<TopDecl>,
+    id: usize,
+}
+
 #[derive(Debug, Property)]
 #[property(get(public))]
 pub struct FieldDecl {
@@ -218,6 +225,15 @@ impl ItemDecl {
     }
 }
 
+impl UnionItemDecl {
+    fn new(top_decl: &Rc<TopDecl>, customize_id: usize) -> Self {
+        Self {
+            typ: Rc::clone(top_decl),
+            id: customize_id,
+        }
+    }
+}
+
 impl FieldDecl {
     fn new(name: &str, top_decl: &Rc<TopDecl>) -> Self {
         Self {

tools/codegen/src/ast/verified/recover.rs

@@ -29,9 +29,13 @@ impl RecoverFromIr for ir::Union {
         if self.items().is_empty() {
             panic!("the union ({}) is empty", self.name());
         }
+
         self.items()
             .iter()
-            .map(|ir_item| deps.get(ir_item.typ()).map(super::ItemDecl::new))
+            .map(|ir_item| {
+                deps.get(ir_item.typ())
+                    .map(|item| super::UnionItemDecl::new(item, ir_item.id()))
+            })
             .collect::<Option<Vec<_>>>()
             .map(|items| {
                 let name = self.name().to_owned();

tools/codegen/src/generator/languages/c/reader.rs

@@ -95,8 +95,12 @@ impl GenReader for ast::Union {
         w!(o, "    inner.ptr = input->ptr + MOL_NUM_T_SIZE;           ");
         w!(o, "    inner.size = input->size - MOL_NUM_T_SIZE;         ");
         w!(o, "    switch(item_id) {{                                 ");
-        for (item_id, item) in self.items().iter().enumerate() {
-            w!(o, "        case {}:                              ", item_id);
+        for item in self.items().iter() {
+            w!(
+                o,
+                "        case {}:                              ",
+                item.id()
+            );
             if item.typ().is_byte() {
                 w!(o, "            return inner.size == 1 ? MOL_OK : MOL_ERR; ");
             } else {