Skip to content

Commit

Permalink
perf: use str instead of Cow to reduce Token size (#958)
Browse files Browse the repository at this point in the history 
* perf: use str instead of Cow to reduce Token size

* fix: fixed lifetime issue when processing tags
  • Loading branch information
@sssooonnnggg
sssooonnnggg authored Jan 4, 2024
1 parent ab70fe6 commit 6ea9523
Show file tree
Hide file tree
Showing 5 changed files with 31 additions and 45 deletions.
16 changes: 2 additions & 14 deletions generator/src/generator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,14 +568,8 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream {
#[cfg(feature = "grammar-extras")]
OptimizedExpr::NodeTag(expr, tag) => {
let expr = generate_expr(*expr);
let tag_cow = {
#[cfg(feature = "std")]
quote! { ::std::borrow::Cow::Borrowed(#tag) }
#[cfg(not(feature = "std"))]
quote! { ::alloc::borrow::Cow::Borrowed(#tag) }
};
quote! {
#expr.and_then(|state| state.tag_node(#tag_cow))
#expr.and_then(|state| state.tag_node(#tag))
}
}
}
Expand Down Expand Up @@ -729,14 +723,8 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
#[cfg(feature = "grammar-extras")]
OptimizedExpr::NodeTag(expr, tag) => {
let expr = generate_expr_atomic(*expr);
let tag_cow = {
#[cfg(feature = "std")]
quote! { ::std::borrow::Cow::Borrowed(#tag) }
#[cfg(not(feature = "std"))]
quote! { ::alloc::borrow::Cow::Borrowed(#tag) }
};
quote! {
#expr.and_then(|state| state.tag_node(#tag_cow))
#expr.and_then(|state| state.tag_node(#tag))
}
}
}
Expand Down
24 changes: 12 additions & 12 deletions pest/src/iterators/pairs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,9 +227,9 @@ impl<'i, R: RuleType> Pairs<'i, R> {
/// state: Box<ParserState<'_, Rule>>,
/// ) -> ParseResult<Box<ParserState<'_, Rule>>> {
/// expr(state, Rule::mul, "*")
/// .and_then(|state| state.tag_node(std::borrow::Cow::Borrowed("mul")))
/// .and_then(|state| state.tag_node("mul"))
/// .or_else(|state| expr(state, Rule::add, "+"))
/// .and_then(|state| state.tag_node(std::borrow::Cow::Borrowed("add")))
/// .and_then(|state| state.tag_node("add"))
/// }
/// fn expr<'a>(
/// state: Box<ParserState<'a, Rule>>,
Expand All @@ -239,10 +239,10 @@ impl<'i, R: RuleType> Pairs<'i, R> {
/// state.rule(r, |state| {
/// state.sequence(|state| {
/// number(state)
/// .and_then(|state| state.tag_node(std::borrow::Cow::Borrowed("lhs")))
/// .and_then(|state| state.tag_node("lhs"))
/// .and_then(|state| state.match_string(o))
/// .and_then(number)
/// .and_then(|state| state.tag_node(std::borrow::Cow::Borrowed("rhs")))
/// .and_then(|state| state.tag_node("rhs"))
/// })
/// })
/// }
Expand Down Expand Up @@ -278,9 +278,9 @@ impl<'i, R: RuleType> Pairs<'i, R> {
/// state: Box<ParserState<'_, Rule>>,
/// ) -> ParseResult<Box<ParserState<'_, Rule>>> {
/// expr(state, Rule::mul, "*")
/// .and_then(|state| state.tag_node(std::borrow::Cow::Borrowed("mul")))
/// .and_then(|state| state.tag_node("mul"))
/// .or_else(|state| expr(state, Rule::add, "+"))
/// .and_then(|state| state.tag_node(std::borrow::Cow::Borrowed("add")))
/// .and_then(|state| state.tag_node("add"))
/// }
/// fn expr<'a>(
/// state: Box<ParserState<'a, Rule>>,
Expand All @@ -290,10 +290,10 @@ impl<'i, R: RuleType> Pairs<'i, R> {
/// state.rule(r, |state| {
/// state.sequence(|state| {
/// number(state)
/// .and_then(|state| state.tag_node(std::borrow::Cow::Borrowed("lhs")))
/// .and_then(|state| state.tag_node("lhs"))
/// .and_then(|state| state.match_string(o))
/// .and_then(number)
/// .and_then(|state| state.tag_node(std::borrow::Cow::Borrowed("rhs")))
/// .and_then(|state| state.tag_node("rhs"))
/// })
/// })
/// }
Expand Down Expand Up @@ -676,9 +676,9 @@ mod tests {
state: Box<ParserState<'_, Rule>>,
) -> ParseResult<Box<ParserState<'_, Rule>>> {
expr(state, Rule::mul, "*")
.and_then(|state| state.tag_node(alloc::borrow::Cow::Borrowed("mul")))
.and_then(|state| state.tag_node("mul"))
.or_else(|state| expr(state, Rule::add, "+"))
.and_then(|state| state.tag_node(alloc::borrow::Cow::Borrowed("add")))
.and_then(|state| state.tag_node("add"))
}
fn expr<'a>(
state: Box<ParserState<'a, Rule>>,
Expand All @@ -688,10 +688,10 @@ mod tests {
state.rule(r, |state| {
state.sequence(|state| {
number(state)
.and_then(|state| state.tag_node(alloc::borrow::Cow::Borrowed("lhs")))
.and_then(|state| state.tag_node("lhs"))
.and_then(|state| state.match_string(o))
.and_then(number)
.and_then(|state| state.tag_node(alloc::borrow::Cow::Borrowed("rhs")))
.and_then(|state| state.tag_node("rhs"))
})
})
}
Expand Down
4 changes: 1 addition & 3 deletions pest/src/iterators/queueable_token.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

use alloc::borrow::Cow;

// This structure serves to improve performance over Token objects in two ways:
//
// * it is smaller than a Token, leading to both less memory use when stored in the queue but also
Expand All @@ -24,7 +22,7 @@ pub enum QueueableToken<'i, R> {
End {
start_token_index: usize,
rule: R,
tag: Option<Cow<'i, str>>,
tag: Option<&'i str>,
input_pos: usize,
},
}
6 changes: 3 additions & 3 deletions pest/src/parser_state.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

use alloc::borrow::{Cow, ToOwned};
use alloc::borrow::ToOwned;
use alloc::boxed::Box;
use alloc::rc::Rc;
use alloc::vec;
Expand Down Expand Up @@ -392,7 +392,7 @@ impl<'i, R: RuleType> ParserState<'i, R> {
/// character(state)
/// .and_then(|state| character(state))
/// .and_then(|state| character(state))
/// .and_then(|state| state.tag_node(std::borrow::Cow::Borrowed("c")))
/// .and_then(|state| state.tag_node("c"))
/// .and_then(|state| character(state))
/// })
/// }
Expand All @@ -407,7 +407,7 @@ impl<'i, R: RuleType> ParserState<'i, R> {
/// assert_eq!(find[0].as_str(), "c")
/// ```
#[inline]
pub fn tag_node(mut self: Box<Self>, tag: Cow<'i, str>) -> ParseResult<Box<Self>> {
pub fn tag_node(mut self: Box<Self>, tag: &'i str) -> ParseResult<Box<Self>> {
if let Some(QueueableToken::End { tag: old, .. }) = self.queue.last_mut() {
*old = Some(tag)
}
Expand Down
26 changes: 13 additions & 13 deletions vm/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,20 +64,20 @@ impl Vm {

/// Runs a parser rule on an input
#[allow(clippy::perf)]
pub fn parse<'a, 'i>(
pub fn parse<'a>(
&'a self,
rule: &'a str,
input: &'i str,
) -> Result<Pairs<'i, &str>, Error<&str>> {
input: &'a str,
) -> Result<Pairs<'a, &str>, Error<&str>> {
pest::state(input, |state| self.parse_rule(rule, state))
}

#[allow(clippy::suspicious)]
fn parse_rule<'a, 'i>(
fn parse_rule<'a>(
&'a self,
rule: &'a str,
state: Box<ParserState<'i, &'a str>>,
) -> ParseResult<Box<ParserState<'i, &'a str>>> {
state: Box<ParserState<'a, &'a str>>,
) -> ParseResult<Box<ParserState<'a, &'a str>>> {
if let Some(ref listener) = self.listener {
if listener(rule.to_owned(), state.position()) {
return Err(ParserState::new(state.position().line_of()));
Expand Down Expand Up @@ -179,11 +179,11 @@ impl Vm {
}
}

fn parse_expr<'a, 'i>(
fn parse_expr<'a>(
&'a self,
expr: &'a OptimizedExpr,
state: Box<ParserState<'i, &'a str>>,
) -> ParseResult<Box<ParserState<'i, &'a str>>> {
state: Box<ParserState<'a, &'a str>>,
) -> ParseResult<Box<ParserState<'a, &'a str>>> {
match *expr {
OptimizedExpr::Str(ref string) => state.match_string(string),
OptimizedExpr::Insens(ref string) => state.match_insensitive(string),
Expand Down Expand Up @@ -245,17 +245,17 @@ impl Vm {
#[cfg(feature = "grammar-extras")]
OptimizedExpr::NodeTag(ref expr, ref tag) => self
.parse_expr(expr, state)
.and_then(|state| state.tag_node(std::borrow::Cow::Owned(tag.clone()))),
.and_then(|state| state.tag_node(tag)),
OptimizedExpr::RestoreOnErr(ref expr) => {
state.restore_on_err(|state| self.parse_expr(expr, state))
}
}
}

fn skip<'a, 'i>(
fn skip<'a>(
&'a self,
state: Box<ParserState<'i, &'a str>>,
) -> ParseResult<Box<ParserState<'i, &'a str>>> {
state: Box<ParserState<'a, &'a str>>,
) -> ParseResult<Box<ParserState<'a, &'a str>>> {
match (
self.rules.contains_key("WHITESPACE"),
self.rules.contains_key("COMMENT"),
Expand Down

0 comments on commit 6ea9523

Please sign in to comment.