make the trivia rules compound atomic under a feature flag

maybe fixes #966

.github/workflows/ci.yml

@@ -46,7 +46,7 @@ jobs:
       - name: cargo clippy
         run: cargo clippy --all --features pretty-print,const_prec_climber,memchr,grammar-extras --all-targets -- -Dwarnings
       - name: cargo test
-        run: cargo test --all --features pretty-print,const_prec_climber,memchr,grammar-extras --release
+        run: cargo test --all --features pretty-print,const_prec_climber,memchr,grammar-extras,inner-trivia --release
       - name: cargo test (ignored)
         run: cargo test -p pest_grammars --lib --verbose --release -- --ignored tests::toml_handles_deep_nesting_unstable
 

debugger/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.7.6"
+version = "2.7.7"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>", "Tomas Tauber <me@tomtau.be>"]
 homepage = "https://pest.rs/"
@@ -14,9 +14,9 @@ readme = "_README.md"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.6" }
-pest_meta = { path = "../meta", version = "2.7.6" }
-pest_vm = { path = "../vm", version = "2.7.6" }
+pest = { path = "../pest", version = "2.7.7" }
+pest_meta = { path = "../meta", version = "2.7.7" }
+pest_vm = { path = "../vm", version = "2.7.7" }
 reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] }
 rustyline = "10"
 serde_json = "1"

derive/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.7.6"
+version = "2.7.7"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -22,8 +22,10 @@ default = ["std"]
 std = ["pest/std", "pest_generator/std"]
 not-bootstrap-in-src = ["pest_generator/not-bootstrap-in-src"]
 grammar-extras = ["pest_generator/grammar-extras"]
+# Makes WHITESPACE and COMMENT rules compound atomic instead of atomic
+inner-trivia = ["pest_generator/inner-trivia"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.7.6", default-features = false }
-pest_generator = { path = "../generator", version = "2.7.6", default-features = false }
+pest = { path = "../pest", version = "2.7.7", default-features = false }
+pest_generator = { path = "../generator", version = "2.7.7", default-features = false }

derive/tests/comment.pest

@@ -0,0 +1,2 @@
+COMMENT = { SingleLineComment }
+SingleLineComment = { "//" ~ (!"\n" ~ ANY) }

derive/tests/comment.rs

@@ -0,0 +1,25 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+extern crate alloc;
+#[cfg(not(feature = "std"))]
+use alloc::{format, vec::Vec};
+
+#[cfg(feature = "inner-trivia")]
+use pest::Parser;
+use pest_derive::Parser;
+
+#[derive(Parser)]
+#[grammar = "../tests/comment.pest"]
+pub struct CommentParser;
+
+#[test]
+#[cfg(feature = "inner-trivia")]
+pub fn test_comment() {
+    let result = CommentParser::parse(Rule::COMMENT, "// some comment\n");
+    assert!(result.is_ok());
+    let mut pairs = result.unwrap();
+    let pair = pairs.next().unwrap();
+    assert_eq!(pair.as_rule(), Rule::COMMENT);
+    let mut inner = pair.into_inner();
+    let comment = inner.next().unwrap();
+    assert_eq!(comment.as_rule(), Rule::SingleLineComment);
+}

generator/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.7.6"
+version = "2.7.7"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -20,10 +20,12 @@ not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"]
 grammar-extras = ["pest_meta/grammar-extras"]
 # Export internal API that is not intended to be stable
 export-internal = []
+# Makes WHITESPACE and COMMENT rules compound atomic instead of atomic
+inner-trivia = []
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.6", default-features = false }
-pest_meta = { path = "../meta", version = "2.7.6" }
+pest = { path = "../pest", version = "2.7.7", default-features = false }
+pest_meta = { path = "../meta", version = "2.7.7" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "2.0"

generator/src/generator.rs

@@ -297,12 +297,18 @@ fn generate_rule(rule: OptimizedRule) -> TokenStream {
         generate_expr_atomic(rule.expr)
     } else if rule.name == "WHITESPACE" || rule.name == "COMMENT" {
         let atomic = generate_expr_atomic(rule.expr);
-
+        #[cfg(not(feature = "inner-trivia"))]
         quote! {
             state.atomic(::pest::Atomicity::Atomic, |state| {
                 #atomic
             })
         }
+        #[cfg(feature = "inner-trivia")]
+        quote! {
+            state.atomic(::pest::Atomicity::CompoundAtomic, |state| {
+                #atomic
+            })
+        }
     } else {
         generate_expr(rule.expr)
     };

grammars/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.7.6"
+version = "2.7.7"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.6" }
-pest_derive = { path = "../derive", version = "2.7.6" }
+pest = { path = "../pest", version = "2.7.7" }
+pest_derive = { path = "../derive", version = "2.7.7" }
 
 [dev-dependencies]
 criterion = "0.5"

meta/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.7.6"
+version = "2.7.7"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.6" }
+pest = { path = "../pest", version = "2.7.7" }
 once_cell = "1.8.0"
 
 [build-dependencies]

pest/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.7.6"
+version = "2.7.7"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"

release.sh

@@ -22,7 +22,13 @@ publish() {
     cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --all-features
   else
     # cannot publish with the `not-bootstrap-in-src` feature enabled
-    cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --features grammar-extras
+    if [ "${1}" = "pest_derive" ] || [ "${1}" = "pest_vm" ]; then
+        cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --features grammar-extras,inner-trivia
+    elif [ "${1}" = "pest_generator" ]; then
+        cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --features grammar-extras,export-internal,inner-trivia
+    else
+        cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --features grammar-extras
+    fi
   fi
   echo ""
 }
@@ -52,4 +58,4 @@ for crate in ${CRATES}; do
   VERSION="$(get_local_version "${crate}")"
   publish "${crate}"
   wait_until_available "${crate}" "${VERSION}"
-done
+done

vm/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_vm"
 description = "pest grammar virtual machine"
-version = "2.7.6"
+version = "2.7.7"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,10 @@ readme = "_README.md"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.6" }
-pest_meta = { path = "../meta", version = "2.7.6" }
+pest = { path = "../pest", version = "2.7.7" }
+pest_meta = { path = "../meta", version = "2.7.7" }
 
 [features]
 grammar-extras = ["pest_meta/grammar-extras"]
+# Makes WHITESPACE and COMMENT rules compound atomic instead of atomic
+inner-trivia = []

vm/src/lib.rs

@@ -129,9 +129,11 @@ impl Vm {
             if rule.name == "WHITESPACE" || rule.name == "COMMENT" {
                 match rule.ty {
                     RuleType::Normal => state.rule(&rule.name, |state| {
-                        state.atomic(Atomicity::Atomic, |state| {
-                            self.parse_expr(&rule.expr, state)
-                        })
+                        #[cfg(feature = "inner-trivia")]
+                        let atomicity = Atomicity::CompoundAtomic;
+                        #[cfg(not(feature = "inner-trivia"))]
+                        let atomicity = Atomicity::Atomic;
+                        state.atomic(atomicity, |state| self.parse_expr(&rule.expr, state))
                     }),
                     RuleType::Silent => state.atomic(Atomicity::Atomic, |state| {
                         self.parse_expr(&rule.expr, state)