diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee630101..3e0a3476 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: - name: cargo clippy run: cargo clippy --all --features pretty-print,const_prec_climber,memchr,grammar-extras --all-targets -- -Dwarnings - name: cargo test - run: cargo test --all --features pretty-print,const_prec_climber,memchr,grammar-extras --release + run: cargo test --all --features pretty-print,const_prec_climber,memchr,grammar-extras,inner-trivia --release - name: cargo test (ignored) run: cargo test -p pest_grammars --lib --verbose --release -- --ignored tests::toml_handles_deep_nesting_unstable diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml index 4e2f67b9..4947d191 100644 --- a/debugger/Cargo.toml +++ b/debugger/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_debugger" description = "pest grammar debugger" -version = "2.7.6" +version = "2.7.7" edition = "2021" authors = ["Dragoș Tiselice ", "Tomas Tauber "] homepage = "https://pest.rs/" @@ -14,9 +14,9 @@ readme = "_README.md" rust-version = "1.61" [dependencies] -pest = { path = "../pest", version = "2.7.6" } -pest_meta = { path = "../meta", version = "2.7.6" } -pest_vm = { path = "../vm", version = "2.7.6" } +pest = { path = "../pest", version = "2.7.7" } +pest_meta = { path = "../meta", version = "2.7.7" } +pest_vm = { path = "../vm", version = "2.7.7" } reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] } rustyline = "10" serde_json = "1" diff --git a/derive/Cargo.toml b/derive/Cargo.toml index c4897b59..f089bc58 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_derive" description = "pest's derive macro" -version = "2.7.6" +version = "2.7.7" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -22,8 +22,10 @@ default = ["std"] std = ["pest/std", "pest_generator/std"] not-bootstrap-in-src = ["pest_generator/not-bootstrap-in-src"] grammar-extras = ["pest_generator/grammar-extras"] +# Makes WHITESPACE and COMMENT rules compound atomic instead of atomic +inner-trivia = ["pest_generator/inner-trivia"] [dependencies] # for tests, included transitively anyway -pest = { path = "../pest", version = "2.7.6", default-features = false } -pest_generator = { path = "../generator", version = "2.7.6", default-features = false } +pest = { path = "../pest", version = "2.7.7", default-features = false } +pest_generator = { path = "../generator", version = "2.7.7", default-features = false } diff --git a/derive/tests/comment.pest b/derive/tests/comment.pest new file mode 100644 index 00000000..499f3a1c --- /dev/null +++ b/derive/tests/comment.pest @@ -0,0 +1,2 @@ +COMMENT = { SingleLineComment } +SingleLineComment = { "//" ~ (!"\n" ~ ANY) } diff --git a/derive/tests/comment.rs b/derive/tests/comment.rs new file mode 100644 index 00000000..cee2526f --- /dev/null +++ b/derive/tests/comment.rs @@ -0,0 +1,25 @@ +#![cfg_attr(not(feature = "std"), no_std)] +extern crate alloc; +#[cfg(not(feature = "std"))] +use alloc::{format, vec::Vec}; + +#[cfg(feature = "inner-trivia")] +use pest::Parser; +use pest_derive::Parser; + +#[derive(Parser)] +#[grammar = "../tests/comment.pest"] +pub struct CommentParser; + +#[test] +#[cfg(feature = "inner-trivia")] +pub fn test_comment() { + let result = CommentParser::parse(Rule::COMMENT, "// some comment\n"); + assert!(result.is_ok()); + let mut pairs = result.unwrap(); + let pair = pairs.next().unwrap(); + assert_eq!(pair.as_rule(), Rule::COMMENT); + let mut inner = pair.into_inner(); + let comment = inner.next().unwrap(); + assert_eq!(comment.as_rule(), Rule::SingleLineComment); +} diff --git a/generator/Cargo.toml b/generator/Cargo.toml index 5e066888..ac0935e1 100644 --- a/generator/Cargo.toml +++ b/generator/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_generator" description = "pest code generator" -version = "2.7.6" +version = "2.7.7" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -20,10 +20,12 @@ not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"] grammar-extras = ["pest_meta/grammar-extras"] # Export internal API that is not intended to be stable export-internal = [] +# Makes WHITESPACE and COMMENT rules compound atomic instead of atomic +inner-trivia = [] [dependencies] -pest = { path = "../pest", version = "2.7.6", default-features = false } -pest_meta = { path = "../meta", version = "2.7.6" } +pest = { path = "../pest", version = "2.7.7", default-features = false } +pest_meta = { path = "../meta", version = "2.7.7" } proc-macro2 = "1.0" quote = "1.0" syn = "2.0" diff --git a/generator/src/generator.rs b/generator/src/generator.rs index f5fa5a1f..de6d8199 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -297,12 +297,18 @@ fn generate_rule(rule: OptimizedRule) -> TokenStream { generate_expr_atomic(rule.expr) } else if rule.name == "WHITESPACE" || rule.name == "COMMENT" { let atomic = generate_expr_atomic(rule.expr); - + #[cfg(not(feature = "inner-trivia"))] quote! { state.atomic(::pest::Atomicity::Atomic, |state| { #atomic }) } + #[cfg(feature = "inner-trivia")] + quote! { + state.atomic(::pest::Atomicity::CompoundAtomic, |state| { + #atomic + }) + } } else { generate_expr(rule.expr) }; diff --git a/grammars/Cargo.toml b/grammars/Cargo.toml index 861c3eba..4f3f2df4 100644 --- a/grammars/Cargo.toml +++ b/grammars/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_grammars" description = "pest popular grammar implementations" -version = "2.7.6" +version = "2.7.7" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -14,8 +14,8 @@ readme = "_README.md" rust-version = "1.61" [dependencies] -pest = { path = "../pest", version = "2.7.6" } -pest_derive = { path = "../derive", version = "2.7.6" } +pest = { path = "../pest", version = "2.7.7" } +pest_derive = { path = "../derive", version = "2.7.7" } [dev-dependencies] criterion = "0.5" diff --git a/meta/Cargo.toml b/meta/Cargo.toml index 8eaf42a6..920e6df2 100644 --- a/meta/Cargo.toml +++ b/meta/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_meta" description = "pest meta language parser and validator" -version = "2.7.6" +version = "2.7.7" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*" rust-version = "1.61" [dependencies] -pest = { path = "../pest", version = "2.7.6" } +pest = { path = "../pest", version = "2.7.7" } once_cell = "1.8.0" [build-dependencies] diff --git a/pest/Cargo.toml b/pest/Cargo.toml index 9473a639..a0f4b7df 100644 --- a/pest/Cargo.toml +++ b/pest/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest" description = "The Elegant Parser" -version = "2.7.6" +version = "2.7.7" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" diff --git a/release.sh b/release.sh index 73cc4206..d06d372c 100755 --- a/release.sh +++ b/release.sh @@ -22,7 +22,13 @@ publish() { cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --all-features else # cannot publish with the `not-bootstrap-in-src` feature enabled - cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --features grammar-extras + if [ "${1}" = "pest_derive" ] || [ "${1}" = "pest_vm" ]; then + cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --features grammar-extras,inner-trivia + elif [ "${1}" = "pest_generator" ]; then + cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --features grammar-extras,export-internal,inner-trivia + else + cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --features grammar-extras + fi fi echo "" } @@ -52,4 +58,4 @@ for crate in ${CRATES}; do VERSION="$(get_local_version "${crate}")" publish "${crate}" wait_until_available "${crate}" "${VERSION}" -done \ No newline at end of file +done diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 820f032f..927b2588 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_vm" description = "pest grammar virtual machine" -version = "2.7.6" +version = "2.7.7" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -14,8 +14,10 @@ readme = "_README.md" rust-version = "1.61" [dependencies] -pest = { path = "../pest", version = "2.7.6" } -pest_meta = { path = "../meta", version = "2.7.6" } +pest = { path = "../pest", version = "2.7.7" } +pest_meta = { path = "../meta", version = "2.7.7" } [features] grammar-extras = ["pest_meta/grammar-extras"] +# Makes WHITESPACE and COMMENT rules compound atomic instead of atomic +inner-trivia = [] diff --git a/vm/src/lib.rs b/vm/src/lib.rs index c3e61d77..da766b8c 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -129,9 +129,11 @@ impl Vm { if rule.name == "WHITESPACE" || rule.name == "COMMENT" { match rule.ty { RuleType::Normal => state.rule(&rule.name, |state| { - state.atomic(Atomicity::Atomic, |state| { - self.parse_expr(&rule.expr, state) - }) + #[cfg(feature = "inner-trivia")] + let atomicity = Atomicity::CompoundAtomic; + #[cfg(not(feature = "inner-trivia"))] + let atomicity = Atomicity::Atomic; + state.atomic(atomicity, |state| self.parse_expr(&rule.expr, state)) }), RuleType::Silent => state.atomic(Atomicity::Atomic, |state| { self.parse_expr(&rule.expr, state)