diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index 270ecf918..f5b351706 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -37,6 +37,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "anyhow" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" + [[package]] name = "arrayvec" version = "0.7.2" @@ -634,11 +640,14 @@ dependencies = [ name = "kclvm-tools" version = "0.1.0" dependencies = [ + "anyhow", "fancy-regex", "indexmap", "kclvm-ast", "kclvm-error", "kclvm-parser", + "kclvm-sema", + "walkdir", ] [[package]] diff --git a/kclvm/ast/Cargo.lock b/kclvm/ast/Cargo.lock index d7fa326c8..dc0cb57cc 100644 --- a/kclvm/ast/Cargo.lock +++ b/kclvm/ast/Cargo.lock @@ -2,24 +2,91 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "annotate-snippets" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78ea013094e5ea606b1c05fe35f1dd7ea1eb1ea259908d040b25bd5ec677ee5" + [[package]] name = "arrayvec" version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bit-set" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.10.2" @@ -29,6 +96,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", +] + [[package]] name = "cc" version = "1.0.73" @@ -47,6 +125,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time", + "winapi", +] + [[package]] name = "cpufeatures" version = "0.2.2" @@ -101,13 +192,22 @@ dependencies = [ "typenum", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "digest" version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ - "block-buffer", + "block-buffer 0.10.2", "crypto-common", ] @@ -126,6 +226,25 @@ dependencies = [ "log", ] +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror", +] + +[[package]] +name = "fancy-regex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6b8560a05112eb52f04b00e5d3790c0dd75d9d980eb8a122fb23b92a623ccf" +dependencies = [ + "bit-set", + "regex", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -135,6 +254,34 @@ dependencies = [ "instant", ] +[[package]] +name = "fixedbitset" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" + +[[package]] +name = "fslock" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + [[package]] name = "generic-array" version = "0.14.5" @@ -145,6 +292,23 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "hashbrown" version = "0.11.2" @@ -180,6 +344,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.1" @@ -195,16 +368,63 @@ dependencies = [ "libc", ] +[[package]] +name = "json_minimal" +version = "0.1.3" + [[package]] name = "kclvm-ast" version = "0.1.0" dependencies = [ + "kclvm-parser", "kclvm-span", "rustc_span", "serde", "serde_json", ] +[[package]] +name = "kclvm-config" +version = "0.1.0" +dependencies = [ + "ahash", + "chrono", + "fslock", + "glob", + "indexmap", + "kclvm-version", + "pathdiff", + "ron", + "rust-crypto", + "serde", + "serde_yaml", + "toml", +] + +[[package]] +name = "kclvm-error" +version = "0.1.0" +dependencies = [ + "annotate-snippets", + "atty", + "indexmap", + "kclvm-runtime", + "kclvm-span", + "rustc_span", + "termcolor", + "termize", + "tracing", +] + +[[package]] +name = "kclvm-lexer" +version = "0.1.0" +dependencies = [ + "kclvm-error", + "rustc_lexer", + "unic-emoji-char", +] + [[package]] name = "kclvm-macros" version = "0.1.0" @@ -215,6 +435,77 @@ dependencies = [ "synstructure", ] +[[package]] +name = "kclvm-parser" +version = "0.1.0" +dependencies = [ + "bstr", + "either", + "enquote", + "kclvm-ast", + "kclvm-config", + "kclvm-error", + "kclvm-lexer", + "kclvm-runtime", + "kclvm-sema", + "kclvm-span", + "num-bigint", + "rustc_data_structures", + "rustc_lexer", + "rustc_span", + "serde", + "serde_json", + "tracing", + "unicode_names2", +] + +[[package]] +name = "kclvm-runtime" +version = "0.1.0" +dependencies = [ + "ahash", + "base64", + "bstr", + "chrono", + "fancy-regex", + "indexmap", + "itertools", + "json_minimal", + "kclvm_runtime_internal_macros", + "libc", + "md5", + "num-integer", + "phf", + "regex", + "serde", + "serde_json", + "serde_yaml", + "sha1", + "sha2 0.9.9", + "unic-ucd-bidi", + "unic-ucd-category", + "unicode-casing", +] + +[[package]] +name = "kclvm-sema" +version = "0.1.0" +dependencies = [ + "ahash", + "bit-set", + "bitflags", + "fancy-regex", + "indexmap", + "kclvm-ast", + "kclvm-error", + "kclvm-runtime", + "kclvm-span", + "once_cell", + "petgraph", + "phf", + "unicode_names2", +] + [[package]] name = "kclvm-span" version = "0.1.0" @@ -224,6 +515,19 @@ dependencies = [ "scoped-tls", ] +[[package]] +name = "kclvm-version" +version = "0.1.0" + +[[package]] +name = "kclvm_runtime_internal_macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -236,6 +540,12 @@ version = "0.2.124" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + [[package]] name = "lock_api" version = "0.4.7" @@ -255,15 +565,33 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "md-5" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" dependencies = [ - "digest", + "digest 0.10.3", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + [[package]] name = "memmap2" version = "0.2.3" @@ -282,6 +610,36 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.13.1" @@ -292,6 +650,18 @@ dependencies = [ "libc", ] +[[package]] +name = "once_cell" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "parking_lot" version = "0.12.0" @@ -315,12 +685,84 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + +[[package]] +name = "petgraph" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ac8b67553a7ca9457ce0e526948cad581819238f4a9d1ea74545851fa24f37" +dependencies = [ + "phf_macros", + "phf_shared", + "proc-macro-hack", +] + +[[package]] +name = "phf_generator" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43f3220d96e0080cc9ea234978ccd80d904eafb17be31bb0f76daaea6493082" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b706f5936eb50ed880ae3009395b43ed19db5bff2ebd459c95e7bf013a89ab86" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68318426de33640f02be62b4ae8eb1261be2efbc337b60c54d845bf4484e0d9" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.37" @@ -348,6 +790,83 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" +dependencies = [ + "libc", + "rand 0.4.6", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.13" @@ -357,6 +876,29 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + +[[package]] +name = "regex-syntax" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -366,6 +908,30 @@ dependencies = [ "winapi", ] +[[package]] +name = "ron" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b861ecaade43ac97886a512b360d01d66be9f41f3c61088b42cedf92e03d678" +dependencies = [ + "base64", + "bitflags", + "serde", +] + +[[package]] +name = "rust-crypto" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" +dependencies = [ + "gcc", + "libc", + "rand 0.3.23", + "rustc-serialize", + "time", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -395,6 +961,12 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rustc-serialize" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" + [[package]] name = "rustc_data_structures" version = "0.0.0" @@ -418,6 +990,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustc_lexer" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86aae0c77166108c01305ee1a36a1e77289d7dc6ca0a3cd91ff4992de2d16a5" +dependencies = [ + "unicode-xid", +] + [[package]] name = "rustc_span" version = "0.0.0" @@ -427,7 +1008,7 @@ dependencies = [ "rustc_data_structures", "scoped-tls", "sha-1", - "sha2", + "sha2 0.10.2", "tracing", "unicode-width", ] @@ -481,6 +1062,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707d15895415db6628332b737c838b88c598522e4dc70647e59b72312924aebc" +dependencies = [ + "indexmap", + "ryu", + "serde", + "yaml-rust", +] + [[package]] name = "sha-1" version = "0.10.0" @@ -489,7 +1082,35 @@ checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.10.3", +] + +[[package]] +name = "sha1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", ] [[package]] @@ -500,9 +1121,15 @@ checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.10.3", ] +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "smallvec" version = "1.8.0" @@ -565,6 +1192,65 @@ dependencies = [ "winapi", ] +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termize" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1706be6b564323ce7092f5f7e6b118a14c8ef7ed0e69c8c5329c914a9f101295" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "thiserror" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi", + "winapi", +] + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + [[package]] name = "tracing" version = "0.1.34" @@ -603,6 +1289,76 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +dependencies = [ + "unic-char-range", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" + +[[package]] +name = "unic-emoji-char" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-bidi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1d568b51222484e1f8209ce48caa6b430bf352962b877d592c29ab31fb53d8c" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-category" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0" +dependencies = [ + "matches", + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +dependencies = [ + "unic-common", +] + +[[package]] +name = "unicode-casing" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56" + [[package]] name = "unicode-width" version = "0.1.9" @@ -615,12 +1371,24 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "unicode_names2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d6678d7916394abad0d4b19df4d3802e1fd84abd7d701f39b75ee71b9e8cf1" + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "winapi" version = "0.3.9" @@ -637,6 +1405,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -685,3 +1462,12 @@ name = "windows_x86_64_msvc" version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] diff --git a/kclvm/ast/Cargo.toml b/kclvm/ast/Cargo.toml index e6cc20a37..46f689666 100644 --- a/kclvm/ast/Cargo.toml +++ b/kclvm/ast/Cargo.toml @@ -11,3 +11,6 @@ serde = { version = "1", features = ["derive"] } serde_json = "1.0" kclvm-span = {path = "../span", version = "0.1.0"} + +[dev-dependencies] +kclvm-parser = {path = "../parser", version = "0.1.0"} diff --git a/kclvm/ast/src/ast.rs b/kclvm/ast/src/ast.rs index 0fe087dc6..999a75029 100644 --- a/kclvm/ast/src/ast.rs +++ b/kclvm/ast/src/ast.rs @@ -166,29 +166,23 @@ impl TryInto> for Node { /// AST node type T pub type NodeRef = Box>; -#[derive(Serialize, Deserialize, Debug, Clone)] -pub enum ParseMode { - Null, - ParseComments, -} - /// KCL command line argument spec, e.g. `kcl main.k -D name=value` -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct CmdArgSpec { pub name: String, pub value: String, } /// KCL command line override spec, e.g. `kcl main.k -O pkgpath:path.to.field=field_value` -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct CmdOverrideSpec { +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct OverrideSpec { pub pkgpath: String, pub field_path: String, pub field_value: String, pub action: OverrideAction, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub enum OverrideAction { CreateOrUpdate, Delete, @@ -201,7 +195,7 @@ pub struct Program { pub main: String, pub pkgs: HashMap>, pub cmd_args: Vec, - pub cmd_overrides: Vec, + pub cmd_overrides: Vec, } impl Program { diff --git a/kclvm/ast/src/config.rs b/kclvm/ast/src/config.rs new file mode 100644 index 000000000..c2a371474 --- /dev/null +++ b/kclvm/ast/src/config.rs @@ -0,0 +1,34 @@ +use crate::ast; + +/// Try get a config expr mut ref from a expr if the expr is a schema or a config. +/// If not, return [None]. +/// TODO: use [TryInto]? +/// +/// # Examples +/// +/// ``` +/// use kclvm_parser::parse_expr; +/// use kclvm_ast::ast; +/// use kclvm_ast::config::try_get_config_expr_mut; +/// +/// let mut expr = parse_expr(r#"{ +/// a: {b: {c = 1}} +/// } +/// "#).unwrap(); +/// assert!(matches!(try_get_config_expr_mut(&mut expr.node), Some(_))); +/// let mut expr = parse_expr(r#"1"#).unwrap(); +/// assert!(matches!(try_get_config_expr_mut(&mut expr.node), None)); +/// ``` +pub fn try_get_config_expr_mut(expr: &mut ast::Expr) -> Option<&mut ast::ConfigExpr> { + match expr { + ast::Expr::Schema(schema_expr) => { + if let ast::Expr::Config(config_expr) = &mut schema_expr.config.node { + Some(config_expr) + } else { + None + } + } + ast::Expr::Config(config_expr) => Some(config_expr), + _ => None + } +} diff --git a/kclvm/ast/src/lib.rs b/kclvm/ast/src/lib.rs index d0cc4279e..edaf80490 100644 --- a/kclvm/ast/src/lib.rs +++ b/kclvm/ast/src/lib.rs @@ -2,6 +2,8 @@ use crate::ast::*; pub mod ast; +pub mod config; +pub mod path; pub mod token; pub mod token_stream; pub mod walker; diff --git a/kclvm/ast/src/path.rs b/kclvm/ast/src/path.rs new file mode 100644 index 000000000..d41fcf8f8 --- /dev/null +++ b/kclvm/ast/src/path.rs @@ -0,0 +1,100 @@ +use crate::ast; + +/// Get config key path from the AST key node and convert string-based AST nodes including +/// `ast::Expr::Identifier` and `ast::Expr::StringLit` to strings. +/// +/// # Examples +/// +/// ``` +/// use kclvm_ast::ast; +/// use kclvm_ast::path::get_key_path; +/// +/// let ident = Some(Box::new(ast::Node::dummy_node(ast::Expr::Identifier(ast::Identifier { +/// names: vec!["alice".to_string()], +/// pkgpath: "".to_string(), +/// ctx: ast::ExprContext::Load, +/// })))); +/// assert_eq!(get_key_path(&ident), "alice"); +/// let str_lit = Some(Box::new(ast::Node::dummy_node(ast::Expr::StringLit(ast::StringLit { +/// is_long_string: false, +/// raw_value: "\"Alice\"".to_string(), +/// value: "Alice".to_string(), +/// })))); +/// assert_eq!(get_key_path(&str_lit), "Alice"); +/// ``` +#[inline] +pub fn get_key_path(key: &Option>) -> String { + match key { + Some(key) => match &key.node { + ast::Expr::Identifier(identifier) => identifier.get_name(), + ast::Expr::StringLit(string_lit) => string_lit.value.clone(), + _ => "".to_string(), + }, + None => "".to_string(), + } +} + +/// Get all attribute paths recursively from a config expression AST node. +/// +/// # Examples +/// +/// ``` +/// use kclvm_parser::parse_expr; +/// use kclvm_ast::ast; +/// use kclvm_ast::path::get_attr_paths_from_config_expr; +/// +/// let expr = parse_expr(r#"{ +/// a: {b: {c = 1}} +/// } +/// "#).unwrap(); +/// if let ast::Expr::Config(config_expr) = &expr.node { +/// assert_eq!(get_attr_paths_from_config_expr(&config_expr), vec![ +/// "a".to_string(), +/// "a.b".to_string(), +/// "a.b.c".to_string(), +/// ]) +/// } else { +/// panic!("invalid config expr {:?}", expr) +/// } +/// ``` +pub fn get_attr_paths_from_config_expr(config: &ast::ConfigExpr) -> Vec { + let mut paths = vec![]; + for entry in &config.items { + let mut entry_paths = get_entry_paths(&entry.node); + paths.append(&mut entry_paths); + } + paths +} + +/// Get all attribute paths from a config entry. +fn get_entry_paths(entry: &ast::ConfigEntry) -> Vec { + let mut paths = vec![]; + let path = get_key_path(&entry.key); + if path.is_empty() || path.trim().is_empty() { + return paths; + } + paths.push(path.clone()); + let option_config_expr = match &entry.value.node { + ast::Expr::Schema(schema_expr) => { + if let ast::Expr::Config(config_expr) = &schema_expr.config.node { + Some(config_expr) + } else { + None + } + } + ast::Expr::Config(config_expr) => Some(config_expr), + _ => None, + }; + if let Some(config_expr) = option_config_expr { + let value_paths = get_attr_paths_from_config_expr(config_expr); + if !value_paths.is_empty() { + paths.append( + &mut value_paths + .iter() + .map(|value_path| format!("{}.{}", path, value_path)) + .collect::>(), + ); + } + } + paths +} diff --git a/kclvm/parser/src/lib.rs b/kclvm/parser/src/lib.rs index d997d5bc4..d93f4ccbd 100644 --- a/kclvm/parser/src/lib.rs +++ b/kclvm/parser/src/lib.rs @@ -89,16 +89,32 @@ pub fn parse_file(filename: &str, code: Option) -> Result ast::NodeRef { - let sm = SourceMap::new(FilePathMapping::empty()); - sm.new_source_file(PathBuf::from("").into(), src.to_string()); - let sess = &ParseSession::with_source_map(Arc::new(sm)); - - create_session_globals_then(|| { - let stream = parse_token_streams(sess, src, BytePos::from_u32(0)); - let mut parser = Parser::new(sess, stream); - parser.parse_expr() - }) +/// Parse a source string to a expression. When input empty string, it will return [None]. +/// +/// # Examples +/// ``` +/// use kclvm_ast::ast; +/// use kclvm_parser::parse_expr; +/// +/// let expr = parse_expr("'alice'").unwrap(); +/// assert!(matches!(expr.node, ast::Expr::StringLit(_))); +/// let expr = parse_expr(""); +/// assert!(matches!(expr, None)); +/// ``` +pub fn parse_expr(src: &str) -> Option> { + if src.is_empty() { + None + } else { + let sm = SourceMap::new(FilePathMapping::empty()); + sm.new_source_file(PathBuf::from("").into(), src.to_string()); + let sess = &ParseSession::with_source_map(Arc::new(sm)); + + Some(create_session_globals_then(|| { + let stream = parse_token_streams(sess, src, BytePos::from_u32(0)); + let mut parser = Parser::new(sess, stream); + parser.parse_expr() + })) + } } #[derive(Debug, Default, Clone)] @@ -107,7 +123,7 @@ pub struct LoadProgramOptions { pub k_code_list: Vec, pub cmd_args: Vec, - pub cmd_overrides: Vec, + pub cmd_overrides: Vec, pub _mode: Option, pub _load_packages: bool, diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index f38efe6fe..dae7bea15 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -18,7 +18,7 @@ pub struct ExecProgramArgs { pub k_code_list: Vec, pub args: Vec, - pub overrides: Vec, + pub overrides: Vec, pub disable_yaml_result: bool, pub print_override_ast: bool, diff --git a/kclvm/sema/src/pre_process/mod.rs b/kclvm/sema/src/pre_process/mod.rs index 58a858bcb..f775b7bd4 100644 --- a/kclvm/sema/src/pre_process/mod.rs +++ b/kclvm/sema/src/pre_process/mod.rs @@ -1,5 +1,6 @@ mod config; mod identifier; +mod multi_assign; use indexmap::IndexMap; use kclvm_ast::ast; @@ -9,6 +10,7 @@ mod tests; pub use config::{fix_config_expr_nest_attr, merge_program}; pub use identifier::{fix_qualified_identifier, fix_raw_identifier_prefix}; +pub use multi_assign::transform_multi_assign; /// Pre-process AST program. pub fn pre_process_program(program: &mut ast::Program) { diff --git a/kclvm/sema/src/pre_process/multi_assign.rs b/kclvm/sema/src/pre_process/multi_assign.rs new file mode 100644 index 000000000..9b3e6faef --- /dev/null +++ b/kclvm/sema/src/pre_process/multi_assign.rs @@ -0,0 +1,96 @@ +use std::collections::HashMap; + +use kclvm_ast::{ast, walker::MutSelfMutWalker}; + +/// Transform AST and split multi target assign statements to multiple assign statements. +/// +/// # Examples +/// +/// ``` +/// use kclvm_parser::parse_file; +/// use kclvm_sema::pre_process::transform_multi_assign; +/// +/// let mut module = parse_file("", Some("a = b = Config {}".to_string())).unwrap(); +/// assert_eq!(module.body.len(), 1); +/// transform_multi_assign(&mut module); +/// assert_eq!(module.body.len(), 2); +/// ``` +pub fn transform_multi_assign(m: &mut ast::Module) { + let mut transformer = MultiAssignTransformer::default(); + transformer.walk_module(m); + let mut insert_count = 0; + for (index, assign_stmt_list) in transformer.multi_assign_mapping { + // Get the origin assign statement insert index in AST module body with offset. + // offset denotes the sum of the number of assigned stmt has been inserted. + let insert_index = index + insert_count; + let pos = match m.body.get(insert_index) { + Some(stmt) => stmt.pos().clone(), + None => bug!("AST module body index {} out of bound", insert_index), + }; + for (insert_offset, assign_stmt) in assign_stmt_list.iter().enumerate() { + // Insert behind the node with the insert offset, so the index plus one. + m.body.insert( + insert_index + insert_offset + 1, + Box::new(ast::Node::node_with_pos( + ast::Stmt::Assign(assign_stmt.clone()), + pos.clone(), + )), + ); + insert_count += 1; + } + } +} + +/// MultiAssignTransformer is used to transform AST Module and split top level +/// multiple target assign statement to multiple assign statements +/// +/// - Before +/// +/// ```kcl +/// a = b = Config {} +/// ``` +/// +/// - After +/// +/// ```kcl +/// a = Config {} +/// b = Config {} +/// ``` +#[derive(Debug, Default)] +struct MultiAssignTransformer { + pub multi_assign_mapping: HashMap>, + pub index: usize, +} + +impl<'ctx> MutSelfMutWalker<'ctx> for MultiAssignTransformer { + fn walk_stmt(&mut self, stmt: &'ctx mut ast::Stmt) { + if let ast::Stmt::Assign(assign_stmt) = stmt { + self.walk_assign_stmt(assign_stmt) + } + // Statement count. + self.index += 1; + } + fn walk_assign_stmt(&mut self, assign_stmt: &'ctx mut ast::AssignStmt) { + if assign_stmt.targets.len() <= 1 { + return; + } + let mut assign_stmt_list = vec![]; + for target in &assign_stmt.targets[1..] { + let mut new_assign_stmt = assign_stmt.clone(); + new_assign_stmt.targets = vec![target.clone()]; + assign_stmt_list.push(new_assign_stmt); + } + self.multi_assign_mapping + .insert(self.index, assign_stmt_list); + assign_stmt.targets = vec![assign_stmt.targets[0].clone()]; + } + fn walk_if_stmt(&mut self, _: &'ctx mut ast::IfStmt) { + // Do not fix AssignStmt in IfStmt + } + fn walk_schema_stmt(&mut self, _: &'ctx mut ast::SchemaStmt) { + // Do not fix AssignStmt in SchemaStmt + } + fn walk_lambda_expr(&mut self, _: &'ctx mut ast::LambdaExpr) { + // Do not fix AssignStmt in LambdaExpr + } +} diff --git a/kclvm/sema/src/pre_process/test_data/multi_assign.k b/kclvm/sema/src/pre_process/test_data/multi_assign.k new file mode 100644 index 000000000..1503add28 --- /dev/null +++ b/kclvm/sema/src/pre_process/test_data/multi_assign.k @@ -0,0 +1,4 @@ +schema Config: + id?: int + +a = b = c = d = Config {} diff --git a/kclvm/sema/src/pre_process/tests.rs b/kclvm/sema/src/pre_process/tests.rs index 3083b88fa..7faef2f31 100644 --- a/kclvm/sema/src/pre_process/tests.rs +++ b/kclvm/sema/src/pre_process/tests.rs @@ -33,3 +33,26 @@ fn test_fix_raw_identifier_prefix() { panic!("invalid assign statement") } } + +#[test] +fn test_transform_multi_assign() { + let targets = ["a", "b", "c", "d"]; + let mut module = parse_file("./src/pre_process/test_data/multi_assign.k", None).unwrap(); + if let ast::Stmt::Assign(assign_stmt) = &module.body[1].node { + assert_eq!(assign_stmt.targets.len(), targets.len()); + for (i, target) in targets.iter().enumerate() { + assert_eq!(assign_stmt.targets[i].node.get_name(), *target); + } + } else { + panic!("invalid assign statement") + } + transform_multi_assign(&mut module); + for (i, target) in targets.iter().enumerate() { + if let ast::Stmt::Assign(assign_stmt) = &module.body[i + 1].node { + assert_eq!(assign_stmt.targets.len(), 1); + assert_eq!(assign_stmt.targets[0].node.get_name(), *target); + } else { + panic!("invalid assign statement") + } + } +} diff --git a/kclvm/src/lib.rs b/kclvm/src/lib.rs index 291f5dc3a..a70c7d951 100644 --- a/kclvm/src/lib.rs +++ b/kclvm/src/lib.rs @@ -70,7 +70,9 @@ pub fn kclvm_cli_run_unsafe(args: *const i8, plugin_agent: *const i8) -> Result< // load ast let mut program = load_program(&files, Some(opts))?; - apply_overrides(&mut program, &args.overrides, &[]); + if let Err(msg) = apply_overrides(&mut program, &args.overrides, &[]) { + return Err(msg.to_string()); + } let scope = resolve_program(&mut program); scope.check_scope_diagnostics(); // gen bc or ll file diff --git a/kclvm/tools/Cargo.lock b/kclvm/tools/Cargo.lock index c878f0d1a..31e99958b 100644 --- a/kclvm/tools/Cargo.lock +++ b/kclvm/tools/Cargo.lock @@ -37,6 +37,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "anyhow" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" + [[package]] name = "arrayvec" version = "0.7.2" @@ -114,6 +120,22 @@ dependencies = [ "lazy_static", "memchr", "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" + +[[package]] +name = "cast" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a" +dependencies = [ + "rustc_version", ] [[package]] @@ -147,6 +169,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "bitflags", + "textwrap", + "unicode-width", +] + [[package]] name = "cpufeatures" version = "0.2.2" @@ -156,6 +189,52 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.1" @@ -201,6 +280,28 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.1.22" @@ -334,6 +435,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.11.2" @@ -378,6 +485,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.1" @@ -393,6 +506,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "json_minimal" version = "0.1.3" @@ -543,12 +665,16 @@ dependencies = [ name = "kclvm-tools" version = "0.1.0" dependencies = [ + "anyhow", + "criterion", "fancy-regex", "indexmap", "kclvm-ast", "kclvm-error", "kclvm-parser", + "kclvm-sema", "pretty_assertions", + "walkdir", ] [[package]] @@ -692,6 +818,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -796,6 +928,34 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "plotters" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" + +[[package]] +name = "plotters-svg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -915,6 +1075,30 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -1070,12 +1254,30 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "ryu" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scoped-tls" version = "1.0.0" @@ -1088,6 +1290,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "semver" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41d061efea015927ac527063765e73601444cdc344ba855bc7bd44578b25e1c" + [[package]] name = "serde" version = "1.0.137" @@ -1097,6 +1305,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.137" @@ -1114,7 +1332,7 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ - "itoa", + "itoa 1.0.1", "ryu", "serde", ] @@ -1268,6 +1486,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.31" @@ -1299,6 +1526,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "toml" version = "0.5.9" @@ -1440,12 +1677,87 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + +[[package]] +name = "web-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/kclvm/tools/Cargo.toml b/kclvm/tools/Cargo.toml index c1b9d3a0b..28a9e64ac 100644 --- a/kclvm/tools/Cargo.toml +++ b/kclvm/tools/Cargo.toml @@ -8,10 +8,20 @@ edition = "2021" [dependencies] indexmap = "1.0" fancy-regex = "0.7.1" +walkdir = "2" +anyhow = "1.0" kclvm-ast = {path = "../ast", version = "0.1.0"} kclvm-error = {path = "../error", version = "0.1.0"} kclvm-parser = {path = "../parser", version = "0.1.0"} +kclvm-sema = {path = "../sema", version = "0.1.0"} [dev-dependencies] pretty_assertions = "1.2.1" +criterion = "0.3" + +[[bench]] +name = "benchmark" +harness = false + + diff --git a/kclvm/tools/benches/benchmark.rs b/kclvm/tools/benches/benchmark.rs new file mode 100644 index 000000000..6dd6efa31 --- /dev/null +++ b/kclvm/tools/benches/benchmark.rs @@ -0,0 +1,18 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use kclvm_tools::query::override_file; + +pub fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("override", |b| { + b.iter(|| { + override_file( + "./benches/test_data/simple.k", + &["config.image=\"image/image:v1\"".to_string()], + &["pkg.to.path".to_string()], + ) + .unwrap(); + }) + }); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/kclvm/tools/benches/test_data/simple.k b/kclvm/tools/benches/test_data/simple.k new file mode 100644 index 000000000..71b4a59e2 --- /dev/null +++ b/kclvm/tools/benches/test_data/simple.k @@ -0,0 +1,14 @@ +import pkg.to.path +schema Data: + id?: int = 0 + value?: str = "value" + +schema Config: + image: str + data?: Data + +config = Config { + image = "image/image:v1" + data = {id = 1, value = "override_value"} +} + diff --git a/kclvm/tools/src/query/mod.rs b/kclvm/tools/src/query/mod.rs index 54781f51d..c21ef12cf 100644 --- a/kclvm/tools/src/query/mod.rs +++ b/kclvm/tools/src/query/mod.rs @@ -1,3 +1,100 @@ +//! This package is mainly the implementation of the KCL query tool, mainly including +//! KCL code modification `override` and other implementations. We can call the `override_file` +//! function to modify the file. The main principle is to parse the AST according to the +//! input file name, and according to the ast::OverrideSpec transforms the nodes in the +//! AST, recursively modifying or deleting the values of the nodes in the AST. pub mod r#override; -pub use r#override::apply_overrides; +#[cfg(test)] +mod tests; +mod util; + +use anyhow::{anyhow, Result}; +use kclvm_ast::ast; +use kclvm_parser::parse_file; + +pub use r#override::{apply_override_on_module, apply_overrides}; + +use self::r#override::parse_override_spec; +use crate::printer::print_ast_module; + +/// Override and rewrite a file with override specifications. Please note that this is an external user API, +/// and it can directly modify the KCL file in place. +/// +/// # Parameters +/// +/// `file`: [&str] +/// The File that need to be overridden +/// +/// `specs`: &\[[String]\] +/// List of specs that need to be overridden. +/// Each spec string satisfies the form: := or :- +/// When the pkgpath is '__main__', `:` can be omitted. +/// +/// `import_paths`: &\[[String]\] +/// List of import paths that are need to be added. +/// +/// # Returns +/// +/// result: [Result] +/// Whether the file has been modified. +/// +/// # Examples +/// +/// ```no_run +/// use kclvm_tools::query::override_file; +/// +/// let result = override_file( +/// "test.k", +/// &["alice.age=18".to_string()], +/// &[] +/// ).unwrap(); +/// ``` +/// +/// - test.k (before override) +/// +/// ```kcl +/// schema Person: +/// age: int +/// +/// alice = Person { +/// age = 10 +/// } +/// ``` +/// +/// - test.k (after override) +/// +/// ```kcl +/// schema Person: +/// age: int +/// +/// alice = Person { +/// age = 18 +/// } +/// ``` +pub fn override_file(file: &str, specs: &[String], import_paths: &[String]) -> Result { + // Parse override spec strings. + let overrides = specs + .iter() + .map(|s| parse_override_spec(s)) + .filter_map(Result::ok) + .collect::>(); + // Parse file to AST module. + let mut module = match parse_file(file, None) { + Ok(module) => module, + Err(msg) => return Err(anyhow!("{}", msg)), + }; + let mut result = false; + // Override AST module. + for o in &overrides { + if apply_override_on_module(&mut module, o, import_paths)? { + result = true; + } + } + // Print AST module. + if result { + let code_str = print_ast_module(&module); + std::fs::write(file, &code_str)? + } + Ok(result) +} diff --git a/kclvm/tools/src/query/override.rs b/kclvm/tools/src/query/override.rs index 638d7f4ea..164fb58d1 100644 --- a/kclvm/tools/src/query/override.rs +++ b/kclvm/tools/src/query/override.rs @@ -1,86 +1,221 @@ +use std::collections::HashSet; + +use anyhow::{anyhow, Result}; + +use kclvm_ast::config::try_get_config_expr_mut; +use kclvm_ast::path::{get_attr_paths_from_config_expr, get_key_path}; use kclvm_ast::walker::MutSelfMutWalker; use kclvm_ast::{ast, walk_if_mut}; use kclvm_parser::parse_expr; +use kclvm_sema::pre_process::{fix_config_expr_nest_attr, transform_multi_assign}; -pub struct OverrideInfo { - pub pkgpath: String, - pub filename: String, - pub module: ast::Module, -} +use crate::printer::print_ast_module; +use super::util::{invalid_spec_error, split_field_path}; + +/// Import statement column offset always start with 1. +/// todo: The (1-based) column offset needs to be constrained by specifications. +const IMPORT_STMT_COLUMN_OFFSET: u64 = 1; + +/// Apply overrides on the AST program with the override specifications. +/// +/// Please note that this a low level internal API used by compiler itself, +/// The parameters of the method are all compiler internal concepts such as +/// AST, etc. +/// +/// # Examples +/// +/// ```no_check +/// use kclvm_parser::load_program; +/// use kclvm_tools::query::r#override::apply_overrides; +/// +/// let mut prog = load_program(&["config.k"], None).unwrap(); +/// let overrides = vec![parse_override_spec("config.id=1").unwrap()]; +/// let import_paths = vec!["path.to.pkg".to_string()]; +/// let result = apply_overrides(&mut prog, &overrides, &import_paths).unwrap(); +/// ``` pub fn apply_overrides( prog: &mut ast::Program, - overrides: &[ast::CmdOverrideSpec], - _import_paths: &[String], -) { + overrides: &[ast::OverrideSpec], + import_paths: &[String], +) -> Result<()> { for o in overrides { let pkgpath = if o.pkgpath.is_empty() { &prog.main } else { &o.pkgpath }; - match prog.pkgs.get_mut(pkgpath) { - Some(modules) => { - for m in modules.iter_mut() { - if fix_module_override(m, o) {} - // module_add_import_paths(m, import_paths) + if let Some(modules) = prog.pkgs.get_mut(pkgpath) { + for m in modules.iter_mut() { + if apply_override_on_module(m, o, import_paths)? { + let code_str = print_ast_module(m); + std::fs::write(&m.filename, &code_str)? } } - None => {} } } + Ok(()) } -pub fn fix_module_override(m: &mut ast::Module, o: &ast::CmdOverrideSpec) -> bool { - let ss = o.field_path.split(".").collect::>(); +/// Apply overrides on the AST module with the override specifications. +/// +/// Please note that this a low level internal API used by compiler itself, +/// The parameters of the method are all compiler internal concepts such as +/// AST, etc. +/// +/// # Examples +/// +/// ```no_check +/// use kclvm_parser::parse_file; +/// use kclvm_tools::query::apply_override_on_module; +/// +/// let mut module = parse_file("", None).unwrap(); +/// let override_spec = parse_override_spec("config.id=1").unwrap(); +/// let import_paths = vec!["path.to.pkg".to_string()]; +/// let result = apply_override_on_module(&mut module, override_spec, &import_paths).unwrap(); +/// ``` +pub fn apply_override_on_module( + m: &mut ast::Module, + o: &ast::OverrideSpec, + import_paths: &[String], +) -> Result { + // Apply import paths on AST module. + apply_import_paths_on_module(m, import_paths)?; + let ss = o.field_path.split('.').collect::>(); if ss.len() <= 1 { - false + return Ok(false); + } + let target_id = ss[0]; + let field = ss[1..].join("."); + let value = &o.field_value; + let key = ast::Identifier { + names: field.split('.').map(|s| s.to_string()).collect(), + ctx: ast::ExprContext::Store, + pkgpath: "".to_string(), + }; + // Transform config expr to simplify the config path query and override. + fix_config_expr_nest_attr(m); + // When there is a multi-target assignment statement of the form `a = b = Config {}`, + // it needs to be transformed into the following form first to prevent the configuration + // from being incorrectly modified. + // ```kcl + // a = Config {} + // b = Config {} + // ``` + transform_multi_assign(m); + let mut transformer = OverrideTransformer { + target_id: target_id.to_string(), + field_path: field, + override_key: key, + override_value: parse_expr(value), + override_target_count: 0, + has_override: false, + action: o.action.clone(), + }; + transformer.walk_module(m); + Ok(transformer.has_override) +} + +/// Parse override spec string to override structure. +/// +/// parse_override_spec("alice.age=10") -> ast::OverrideSpec { +/// pkgpath: "".to_string(), +/// field_path: "alice.age".to_string(), +/// field_value: "10".to_string(), +/// action: ast::OverrideAction::CreateOrUpdate, +/// } +pub(crate) fn parse_override_spec(spec: &str) -> Result { + if spec.contains('=') { + // Create or update the override value. + let split_values = spec.splitn(2, '=').collect::>(); + let path = split_values + .get(0) + .ok_or_else(|| invalid_spec_error(spec))?; + let field_value = split_values + .get(1) + .ok_or_else(|| invalid_spec_error(spec))?; + let (pkgpath, field_path) = split_field_path(path)?; + Ok(ast::OverrideSpec { + pkgpath, + field_path, + field_value: field_value.to_string(), + action: ast::OverrideAction::CreateOrUpdate, + }) + } else if let Some(stripped_spec) = spec.strip_suffix('-') { + // Delete the override value. + let (pkgpath, field_path) = split_field_path(stripped_spec)?; + Ok(ast::OverrideSpec { + pkgpath, + field_path, + field_value: "".to_string(), + action: ast::OverrideAction::Delete, + }) } else { - let target_id = ss[0]; - let field = ss[1..].join("."); - let value = &o.field_value; - let key = ast::Identifier { - names: field.split(".").map(|s| s.to_string()).collect(), - ctx: ast::ExprContext::Store, - pkgpath: "".to_string(), - }; - let val = build_node_from_string(value); - let mut transformer = OverrideTransformer { - target_id: target_id.to_string(), - field_path: field, - override_key: key, - override_value: val, - override_target_count: 0, - has_override: false, - action: o.action.clone(), - }; - transformer.walk_module(m); - transformer.has_override + Err(invalid_spec_error(spec)) } } -pub fn build_node_from_string(value: &str) -> ast::NodeRef { - let expr = parse_expr(value); - expr +// Transform the AST module with the import path list. +fn apply_import_paths_on_module(m: &mut ast::Module, import_paths: &[String]) -> Result<()> { + if import_paths.is_empty() { + return Ok(()); + } + let mut exist_import_set: HashSet = HashSet::new(); + for stmt in &m.body { + if let ast::Stmt::Import(import_stmt) = &stmt.node { + exist_import_set.insert(import_stmt.path.to_string()); + } + } + for (i, path) in import_paths.iter().enumerate() { + let line: u64 = i as u64 + 1; + if exist_import_set.contains(path) { + continue; + } + let name = path + .split('.') + .last() + .ok_or_else(|| anyhow!("Invalid import path {}", path))?; + let import_node = ast::ImportStmt { + path: path.to_string(), + rawpath: "".to_string(), + name: name.to_string(), + asname: None, + }; + let import_stmt = Box::new(ast::Node::new( + ast::Stmt::Import(import_node), + m.filename.clone(), + line, + IMPORT_STMT_COLUMN_OFFSET, + line, + // i denotes the space len between the `import` keyword and the path. + ("import".len() + path.len() + 1) as u64, + )); + m.body.insert((line - 1) as usize, import_stmt) + } + Ok(()) } -pub struct OverrideTransformer { +/// OverrideTransformer is used to walk AST and transform it with the override values. +struct OverrideTransformer { pub target_id: String, pub field_path: String, pub override_key: ast::Identifier, - pub override_value: ast::NodeRef, + pub override_value: Option>, pub override_target_count: usize, pub has_override: bool, pub action: ast::OverrideAction, } impl<'ctx> MutSelfMutWalker<'ctx> for OverrideTransformer { - fn walk_schema_stmt(&mut self, _: &'ctx mut ast::SchemaStmt) { - // Do not override AssignStmt in SchemaStmt - } - fn walk_unification_stmt(&mut self, unification_stmt: &'ctx mut ast::UnificationStmt) { - if unification_stmt.target.node.names[0] != self.target_id { + let name = match unification_stmt.target.node.names.get(0) { + Some(name) => name, + None => bug!( + "Invalid AST unification target names {:?}", + unification_stmt.target.node.names + ), + }; + if name != &self.target_id { return; } self.override_target_count = 1; @@ -112,20 +247,22 @@ impl<'ctx> MutSelfMutWalker<'ctx> for OverrideTransformer { if self.override_target_count == 0 { return; } - if true { - // Not exist and append an override value when the action is CREATE_OR_UPDATE - if let ast::OverrideAction::CreateOrUpdate = self.action { - if let ast::Expr::Config(config_expr) = &mut schema_expr.config.node { - config_expr - .items - .push(Box::new(ast::Node::dummy_node(ast::ConfigEntry { - key: Some(Box::new(ast::Node::dummy_node(ast::Expr::Identifier( - self.override_key.clone(), - )))), - value: self.override_value.clone(), - operation: ast::ConfigEntryOperation::Override, - insert_index: -1, - }))); + if let ast::Expr::Config(config_expr) = &mut schema_expr.config.node { + if !self.lookup_config_and_replace(config_expr) { + // Not exist and append an override value when the action is CREATE_OR_UPDATE + if let ast::OverrideAction::CreateOrUpdate = self.action { + if let ast::Expr::Config(config_expr) = &mut schema_expr.config.node { + config_expr + .items + .push(Box::new(ast::Node::dummy_node(ast::ConfigEntry { + key: Some(Box::new(ast::Node::dummy_node(ast::Expr::Identifier( + self.override_key.clone(), + )))), + value: self.clone_override_value(), + operation: ast::ConfigEntryOperation::Override, + insert_index: -1, + }))); + } } } } @@ -138,44 +275,113 @@ impl<'ctx> MutSelfMutWalker<'ctx> for OverrideTransformer { self.walk_expr(&mut config_entry.node.value.node); } } + + fn walk_if_stmt(&mut self, _: &'ctx mut ast::IfStmt) { + // Do not override AssignStmt in IfStmt + } + fn walk_schema_stmt(&mut self, _: &'ctx mut ast::SchemaStmt) { + // Do not override AssignStmt in SchemaStmt + } + fn walk_lambda_expr(&mut self, _: &'ctx mut ast::LambdaExpr) { + // Do not override AssignStmt in LambdaExpr + } } impl OverrideTransformer { - pub(crate) fn _get_schema_config_field_paths( - &mut self, - schema_expr: &mut ast::SchemaExpr, - ) -> (Vec, Vec) { - if let ast::Expr::Config(config_expr) = &mut schema_expr.config.node { - self._get_config_field_paths(config_expr) - } else { - (vec![], vec![]) + /// Lookup schema config all fields and replace if it is matched with the override spec, + /// return whether is found a replaced one. + fn lookup_config_and_replace(&mut self, config_expr: &mut ast::ConfigExpr) -> bool { + // Get all entry paths from a config expression. + let paths = get_attr_paths_from_config_expr(config_expr); + // Query whether there is a matching path from the path lookup table. + match paths.iter().position(|r| r == &self.field_path) { + Some(pos) => { + let path = &paths[pos]; + // Split a path into multiple parts. `a.b.c` -> ["a", "b", "c"] + let parts = path.split('.').collect::>(); + self.replace_config_with_path_parts(config_expr, &parts); + true + } + None => false, } } - pub(crate) fn _get_config_field_paths( + + /// Replace AST config expr with one part of path. The implementation of this function + /// uses recursive matching to find the config entry need to be modified. + fn replace_config_with_path_parts( &mut self, - config: &mut ast::ConfigExpr, - ) -> (Vec, Vec) { - let mut paths = vec![]; - let mut paths_with_id = vec![]; - for entry in config.items.iter_mut() { - let (mut _paths, mut _paths_with_id) = self._get_key_value_paths(&mut entry.node); - paths.append(&mut _paths); - paths_with_id.append(&mut &mut _paths_with_id); + config_expr: &mut ast::ConfigExpr, + parts: &[&str], + ) { + // Do not replace empty path parts and out of index parts on the config expression. + if parts.is_empty() { + return; + } + // Always take the first part to match, because recursive search is required. + let part = parts[0]; + let mut delete_index_set = HashSet::new(); + // Loop all entries in the config expression and replace, because there may be duplicate + // configuration items in config. + for (i, item) in config_expr.items.iter_mut().enumerate() { + // Compare each field of the config structure one by one. + // - `part` denotes the path entered by the user to be modified. + // - `get_path_key` returns the real config key name. + // For example, the real config node is `a: {b: c: {}}`. The path + // that needs to be modified is `a.b.c`, and its parts are ["a", "b", "c"]. + if part == get_key_path(&item.node.key) { + // When the last part of the path is successfully recursively matched, + // it indicates that the original value that needs to be overwritten + // is successfully found, and the new value is used to overwrite it. + // - `parts.len() == 1` denotes the path matches exactly. + if parts.len() == 1 { + match self.action { + ast::OverrideAction::CreateOrUpdate => { + let mut value = self.clone_override_value(); + // Use position information that needs to override the expression. + value.set_pos(item.pos()); + // Override the node value. + item.node.value = value; + } + ast::OverrideAction::Delete => { + // Store the config entry delete index into the delete index set. + // Because we can't delete the entry directly in the loop + delete_index_set.insert(i); + } + } + } + // Replace value recursively using the path composed by subsequent parts. + // + // The reason for using recursion instead of looping for path matching + // is that rust cannot directly hold shared references to AST nodes + // (ast::NodeRef is a Box), so recursive search is performed + // directly on AST nodes. + else if let Some(config_expr) = try_get_config_expr_mut(&mut item.node.value.node) + { + self.replace_config_with_path_parts(config_expr, &parts[1..]); + } + } + } + // Delete entries according delete index set. + if !delete_index_set.is_empty() { + let items: Vec<(usize, &ast::NodeRef)> = config_expr + .items + .iter() + .enumerate() + .filter(|(i, _)| !delete_index_set.contains(i)) + .collect(); + config_expr.items = items + .iter() + .map(|(_, item)| <&ast::NodeRef>::clone(item).clone()) + .collect(); } - (paths, paths_with_id) - } - pub(crate) fn _get_key_value_paths( - &mut self, - _entry: &mut ast::ConfigEntry, - ) -> (Vec, Vec) { - (vec![], vec![]) } - pub(crate) fn _find_schema_config_and_repalce( - &mut self, - _schema_config: &mut ast::SchemaExpr, - _field_path: &str, - _value: &ast::NodeRef, - ) -> bool { - false + + /// Clone a override value + #[inline] + fn clone_override_value(&mut self) -> ast::NodeRef { + match &self.override_value { + Some(v) => v.clone(), + None => bug!("Override value is None"), + } } } diff --git a/kclvm/tools/src/query/test_data/config.k b/kclvm/tools/src/query/test_data/config.k new file mode 100644 index 000000000..a29c47a19 --- /dev/null +++ b/kclvm/tools/src/query/test_data/config.k @@ -0,0 +1,59 @@ +schema Main: + name?: str + env?: [{str:}] + +schema Probe: + initialDelaySeconds?: int + timeoutSeconds?: int + periodSeconds?: int = 10 + successThreshold?: int + failureThreshold?: int + +schema AppConfiguration: + appName: str + image: str + overQuota: bool = False + resource: {str:} + mainContainer?: Main + labels: {str:} + probe?: Probe + +appConfiguration = AppConfiguration { + appName: "kclvm" + image: "kclvm/kclvm:v0.1.0" + resource: { + cpu: "4" + disk: "50Gi" + memory: "12Gi" + } + labels: { + key: { + key: "value" + } + } + mainContainer: Main { + name: "kclvm" + } + overQuota = True + overQuota = True + probe: Probe {} +} + +appConfigurationUnification: AppConfiguration { + appName: "kclvm" + image: "kclvm/kclvm:v0.1.0" + resource: { + cpu: "4" + disk: "50Gi" + memory: "12Gi" + } + labels: { + key: { + key: "value" + } + } + mainContainer: Main { + name: "kclvm" + } + overQuota: True +} diff --git a/kclvm/tools/src/query/test_data/import_paths.k b/kclvm/tools/src/query/test_data/import_paths.k new file mode 100644 index 000000000..80031ce77 --- /dev/null +++ b/kclvm/tools/src/query/test_data/import_paths.k @@ -0,0 +1,8 @@ +import pkg.pkg +import pkg +schema Data: + id?: int = 0 + value?: str = "value" + +data = Data {value = "override_value"} + diff --git a/kclvm/tools/src/query/test_data/kcl.mod b/kclvm/tools/src/query/test_data/kcl.mod new file mode 100644 index 000000000..e69de29bb diff --git a/kclvm/tools/src/query/test_data/simple.k b/kclvm/tools/src/query/test_data/simple.k new file mode 100644 index 000000000..99d39a2ae --- /dev/null +++ b/kclvm/tools/src/query/test_data/simple.k @@ -0,0 +1,18 @@ +schema Data: + id?: int = 0 + value?: str = "value" + +schema Config: + image: str + data?: Data + +if True: + configOther = Config {image = "image/other:v1"} + + +config = Config { + image = "image/image:v1" + data = {id = 1, value = "override_value"} + data = {id = 1, value = "override_value"} +} + diff --git a/kclvm/tools/src/query/tests.rs b/kclvm/tools/src/query/tests.rs new file mode 100644 index 000000000..7915242f3 --- /dev/null +++ b/kclvm/tools/src/query/tests.rs @@ -0,0 +1,118 @@ +use super::{r#override::apply_override_on_module, *}; +use crate::printer::print_ast_module; +use kclvm_ast::ast; +use kclvm_parser::parse_file; +use pretty_assertions::assert_eq; + +/// Test override_file result. +#[test] +fn test_override_file_simple() { + let specs = vec![ + "config.image=\"image/image\"".to_string(), + ":config.image=\"image/image:v1\"".to_string(), + ":config.data={id=1,value=\"override_value\"}".to_string(), + ]; + let import_paths = vec![]; + assert_eq!( + override_file("./src/query/test_data/simple.k", &specs, &import_paths).unwrap(), + true + ) +} +/// Test override_file result. +#[test] +fn test_override_file_import_paths() { + let specs = vec!["data.value=\"override_value\"".to_string()]; + let import_paths = vec!["pkg".to_string(), "pkg.pkg".to_string()]; + assert_eq!( + override_file( + "./src/query/test_data/import_paths.k", + &specs, + &import_paths + ) + .unwrap(), + true + ) +} + +/// Test override_file result with the expected modified AST. +#[test] +fn test_override_file_config() { + let specs = vec![ + "appConfiguration.image=\"kcl/kcl:{}\".format(version)".to_string(), + "appConfiguration.mainContainer.name=\"override_name\"".to_string(), + "appConfiguration.labels.key.key=\"override_value\"".to_string(), + "appConfiguration.overQuota=False".to_string(), + "appConfiguration.probe={periodSeconds=20}".to_string(), + "appConfiguration.resource-".to_string(), + "appConfigurationUnification.image=\"kcl/kcl:v0.1\"".to_string(), + "appConfigurationUnification.mainContainer.name=\"override_name\"".to_string(), + "appConfigurationUnification.labels.key.key=\"override_value\"".to_string(), + "appConfigurationUnification.overQuota=False".to_string(), + "appConfigurationUnification.resource.cpu-".to_string(), + ]; + let overrides = specs + .iter() + .map(|s| parse_override_spec(s)) + .filter_map(Result::ok) + .collect::>(); + let import_paths = vec![]; + let mut module = parse_file("./src/query/test_data/config.k", None).unwrap(); + for o in &overrides { + apply_override_on_module(&mut module, o, &import_paths).unwrap(); + } + let expected_code = print_ast_module(&module); + assert_eq!( + expected_code, + r#"schema Main: + name?: str + env?: [{str:}] + +schema Probe: + initialDelaySeconds?: int + timeoutSeconds?: int + periodSeconds?: int = 10 + successThreshold?: int + failureThreshold?: int + +schema AppConfiguration: + appName: str + image: str + overQuota: bool = False + resource: {str:} + mainContainer?: Main + labels: {str:} + probe?: Probe + +appConfiguration = AppConfiguration { + appName: "kclvm" + image: "kcl/kcl:{}".format(version) + labels: {key: {key: "override_value"}} + mainContainer: Main {name: "override_name"} + overQuota = False + overQuota = False + probe: {periodSeconds = 20} +} + +appConfigurationUnification: AppConfiguration { + appName: "kclvm" + image: "kcl/kcl:v0.1" + resource: { + disk: "50Gi" + memory: "12Gi" + } + labels: {key: {key: "override_value"}} + mainContainer: Main {name: "override_name"} + overQuota: False +} +"# + ); +} + +/// Test override spec parser. +#[test] +fn test_parse_override_spec_invalid() { + let specs = vec![":a:", "=a=", ":a", "a-1"]; + for spec in specs { + assert!(parse_override_spec(spec).is_err(), "{} test failed", spec); + } +} diff --git a/kclvm/tools/src/query/util.rs b/kclvm/tools/src/query/util.rs new file mode 100644 index 000000000..3d7bf3b90 --- /dev/null +++ b/kclvm/tools/src/query/util.rs @@ -0,0 +1,29 @@ +use anyhow::{anyhow, Result}; + +/// Get field package path and identifier name from the path. +/// (TODO: Needs to be a package related to the language specification +/// and move this function into it.) +/// +/// split_field_path("pkg.to.path:field") -> ("pkg.to.path", "field") +pub(crate) fn split_field_path(path: &str) -> Result<(String, String)> { + let err = Err(anyhow!("Invalid field path {:?}", path)); + let paths = path.splitn(2, ':').collect::>(); + let (pkgpath, field_path) = if paths.len() == 1 { + ("".to_string(), paths[0].to_string()) + } else if paths.len() == 2 { + (paths[0].to_string(), paths[1].to_string()) + } else { + return err; + }; + if field_path.is_empty() { + err + } else { + Ok((pkgpath, field_path)) + } +} + +/// Get the invalid spec error message. +#[inline] +pub(crate) fn invalid_spec_error(spec: &str) -> anyhow::Error { + anyhow!("Invalid spec format '{}', expected := or :-", spec) +}