From aec7e11b531787a8d13c2fac36ad90a5d42c4281 Mon Sep 17 00:00:00 2001 From: marcfir <72923599+marcfir@users.noreply.github.com> Date: Thu, 4 Jan 2024 20:30:53 +0000 Subject: [PATCH 1/2] feat(generator): Export generator and docs api Signed-off-by: marcfir <72923599+marcfir@users.noreply.github.com> --- generator/src/docs.rs | 8 ++++++-- generator/src/generator.rs | 7 ++++++- generator/src/lib.rs | 19 ++++++++++++------- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/generator/src/docs.rs b/generator/src/docs.rs index ccc82e77..22f4f312 100644 --- a/generator/src/docs.rs +++ b/generator/src/docs.rs @@ -1,9 +1,13 @@ +//! Type and helper to collect the gramamr and rule documentation. + use pest::iterators::Pairs; use pest_meta::parser::Rule; use std::collections::HashMap; +/// Abstraction for the grammer and rule doc. #[derive(Debug)] -pub(crate) struct DocComment { +pub struct DocComment { + /// The grammar documentation is defined at the beginning of a file with //!. pub grammar_doc: String, /// HashMap for store all doc_comments for rules. @@ -33,7 +37,7 @@ pub(crate) struct DocComment { /// grammar_doc = "This is a grammar doc" /// line_docs = { "foo": "line doc 1\nline doc 2", "bar": "line doc 3" } /// ``` -pub(crate) fn consume(pairs: Pairs<'_, Rule>) -> DocComment { +pub fn consume(pairs: Pairs<'_, Rule>) -> DocComment { let mut grammar_doc = String::new(); let mut line_docs: HashMap = HashMap::new(); diff --git a/generator/src/generator.rs b/generator/src/generator.rs index 7a527c5a..b06454c5 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -7,6 +7,8 @@ // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. +//! Helpers to generate the code for the Parser `derive``. + use std::path::PathBuf; use proc_macro2::TokenStream; @@ -20,7 +22,10 @@ use pest_meta::optimizer::*; use crate::docs::DocComment; use crate::ParsedDerive; -pub(crate) fn generate( +/// Generates the corresponding parser based based on the processed macro input. If `include_grammar` +/// is set to true, it'll generate an explicit "include_str" statement (done in pest_derive, but +/// turned off in the local bootstrap). +pub fn generate( parsed_derive: ParsedDerive, paths: Vec, rules: Vec, diff --git a/generator/src/lib.rs b/generator/src/lib.rs index cbd13eaf..9e82699b 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -26,13 +26,14 @@ use std::fs::File; use std::io::{self, Read}; use std::path::Path; +use generator::generate; use proc_macro2::TokenStream; use syn::{Attribute, DeriveInput, Expr, ExprLit, Generics, Ident, Lit, Meta}; #[macro_use] mod macros; -mod docs; -mod generator; +pub mod docs; +pub mod generator; use pest_meta::parser::{self, rename_meta_rule, Rule}; use pest_meta::{optimizer, unwrap_or_report, validator}; @@ -96,7 +97,7 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { let ast = unwrap_or_report(parser::consume_rules(pairs)); let optimized = optimizer::optimize(ast); - generator::generate( + generate( parsed_derive, paths, optimized, @@ -119,10 +120,14 @@ enum GrammarSource { Inline(String), } -struct ParsedDerive { - pub(crate) name: Ident, - pub(crate) generics: Generics, - pub(crate) non_exhaustive: bool, +/// Parsed information of the derive and the attributes. +pub struct ParsedDerive { + /// The identifier of the deriving struct, union, or enum. + pub name: Ident, + /// The generics of the deriving struct, union, or enum. + pub generics: Generics, + /// Indicates whether the 'non_exhaustive' attribute is added to the 'Rule' enum. + pub non_exhaustive: bool, } fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec) { From e657009a1735660d0e1c55f15a698e54e27d8107 Mon Sep 17 00:00:00 2001 From: marcfir <72923599+marcfir@users.noreply.github.com> Date: Fri, 5 Jan 2024 12:33:32 +0000 Subject: [PATCH 2/2] feat(generator): Add feature guard for export and restructure modules * Make internal API public with the `export-internal` feature. * Move the parse_derive functions and types to a new `parse_derive module. Signed-off-by: marcfir <72923599+marcfir@users.noreply.github.com> --- generator/Cargo.toml | 2 + generator/src/generator.rs | 2 +- generator/src/lib.rs | 172 +++------------------------------- generator/src/parse_derive.rs | 172 ++++++++++++++++++++++++++++++++++ 4 files changed, 190 insertions(+), 158 deletions(-) create mode 100644 generator/src/parse_derive.rs diff --git a/generator/Cargo.toml b/generator/Cargo.toml index 0bdf4bd4..5e066888 100644 --- a/generator/Cargo.toml +++ b/generator/Cargo.toml @@ -18,6 +18,8 @@ default = ["std"] std = ["pest/std"] not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"] grammar-extras = ["pest_meta/grammar-extras"] +# Export internal API that is not intended to be stable +export-internal = [] [dependencies] pest = { path = "../pest", version = "2.7.6", default-features = false } diff --git a/generator/src/generator.rs b/generator/src/generator.rs index b06454c5..f5fa5a1f 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -20,7 +20,7 @@ use pest_meta::ast::*; use pest_meta::optimizer::*; use crate::docs::DocComment; -use crate::ParsedDerive; +use crate::parse_derive::ParsedDerive; /// Generates the corresponding parser based based on the processed macro input. If `include_grammar` /// is set to true, it'll generate an explicit "include_str" statement (done in pest_derive, but diff --git a/generator/src/lib.rs b/generator/src/lib.rs index 9e82699b..6018ce47 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -28,13 +28,27 @@ use std::path::Path; use generator::generate; use proc_macro2::TokenStream; -use syn::{Attribute, DeriveInput, Expr, ExprLit, Generics, Ident, Lit, Meta}; +use syn::DeriveInput; #[macro_use] mod macros; + +#[cfg(feature = "export-internal")] pub mod docs; +#[cfg(not(feature = "export-internal"))] +mod docs; + +#[cfg(feature = "export-internal")] pub mod generator; +#[cfg(not(feature = "export-internal"))] +mod generator; + +#[cfg(feature = "export-internal")] +pub mod parse_derive; +#[cfg(not(feature = "export-internal"))] +mod parse_derive; +use crate::parse_derive::{parse_derive, GrammarSource}; use pest_meta::parser::{self, rename_meta_rule, Rule}; use pest_meta::{optimizer, unwrap_or_report, validator}; @@ -114,164 +128,8 @@ fn read_file>(path: P) -> io::Result { Ok(string) } -#[derive(Debug, PartialEq)] -enum GrammarSource { - File(String), - Inline(String), -} - -/// Parsed information of the derive and the attributes. -pub struct ParsedDerive { - /// The identifier of the deriving struct, union, or enum. - pub name: Ident, - /// The generics of the deriving struct, union, or enum. - pub generics: Generics, - /// Indicates whether the 'non_exhaustive' attribute is added to the 'Rule' enum. - pub non_exhaustive: bool, -} - -fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec) { - let name = ast.ident; - let generics = ast.generics; - - let grammar: Vec<&Attribute> = ast - .attrs - .iter() - .filter(|attr| { - let path = attr.meta.path(); - path.is_ident("grammar") || path.is_ident("grammar_inline") - }) - .collect(); - - if grammar.is_empty() { - panic!("a grammar file needs to be provided with the #[grammar = \"PATH\"] or #[grammar_inline = \"GRAMMAR CONTENTS\"] attribute"); - } - - let mut grammar_sources = Vec::with_capacity(grammar.len()); - for attr in grammar { - grammar_sources.push(get_attribute(attr)) - } - - let non_exhaustive = ast - .attrs - .iter() - .any(|attr| attr.meta.path().is_ident("non_exhaustive")); - - ( - ParsedDerive { - name, - generics, - non_exhaustive, - }, - grammar_sources, - ) -} - -fn get_attribute(attr: &Attribute) -> GrammarSource { - match &attr.meta { - Meta::NameValue(name_value) => match &name_value.value { - Expr::Lit(ExprLit { - lit: Lit::Str(string), - .. - }) => { - if name_value.path.is_ident("grammar") { - GrammarSource::File(string.value()) - } else { - GrammarSource::Inline(string.value()) - } - } - _ => panic!("grammar attribute must be a string"), - }, - _ => panic!("grammar attribute must be of the form `grammar = \"...\"`"), - } -} - #[cfg(test)] mod tests { - use super::parse_derive; - use super::GrammarSource; - - #[test] - fn derive_inline_file() { - let definition = " - #[other_attr] - #[grammar_inline = \"GRAMMAR\"] - pub struct MyParser<'a, T>; - "; - let ast = syn::parse_str(definition).unwrap(); - let (_, filenames) = parse_derive(ast); - assert_eq!(filenames, [GrammarSource::Inline("GRAMMAR".to_string())]); - } - - #[test] - fn derive_ok() { - let definition = " - #[other_attr] - #[grammar = \"myfile.pest\"] - pub struct MyParser<'a, T>; - "; - let ast = syn::parse_str(definition).unwrap(); - let (parsed_derive, filenames) = parse_derive(ast); - assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); - assert!(!parsed_derive.non_exhaustive); - } - - #[test] - fn derive_multiple_grammars() { - let definition = " - #[other_attr] - #[grammar = \"myfile1.pest\"] - #[grammar = \"myfile2.pest\"] - pub struct MyParser<'a, T>; - "; - let ast = syn::parse_str(definition).unwrap(); - let (_, filenames) = parse_derive(ast); - assert_eq!( - filenames, - [ - GrammarSource::File("myfile1.pest".to_string()), - GrammarSource::File("myfile2.pest".to_string()) - ] - ); - } - - #[test] - fn derive_nonexhaustive() { - let definition = " - #[non_exhaustive] - #[grammar = \"myfile.pest\"] - pub struct MyParser<'a, T>; - "; - let ast = syn::parse_str(definition).unwrap(); - let (parsed_derive, filenames) = parse_derive(ast); - assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); - assert!(parsed_derive.non_exhaustive); - } - - #[test] - #[should_panic(expected = "grammar attribute must be a string")] - fn derive_wrong_arg() { - let definition = " - #[other_attr] - #[grammar = 1] - pub struct MyParser<'a, T>; - "; - let ast = syn::parse_str(definition).unwrap(); - parse_derive(ast); - } - - #[test] - #[should_panic( - expected = "a grammar file needs to be provided with the #[grammar = \"PATH\"] or #[grammar_inline = \"GRAMMAR CONTENTS\"] attribute" - )] - fn derive_no_grammar() { - let definition = " - #[other_attr] - pub struct MyParser<'a, T>; - "; - let ast = syn::parse_str(definition).unwrap(); - parse_derive(ast); - } #[doc = "Matches dar\n\nMatch dar description\n"] #[test] diff --git a/generator/src/parse_derive.rs b/generator/src/parse_derive.rs new file mode 100644 index 00000000..52374c2a --- /dev/null +++ b/generator/src/parse_derive.rs @@ -0,0 +1,172 @@ +// pest. The Elegant Parser +// Copyright (c) 2018 Dragoș Tiselice +// +// Licensed under the Apache License, Version 2.0 +// or the MIT +// license , at your +// option. All files in the project carrying such notice may not be copied, +// modified, or distributed except according to those terms. + +//! Types and helpers to parse the input of the derive macro. + +use syn::{Attribute, DeriveInput, Expr, ExprLit, Generics, Ident, Lit, Meta}; + +#[derive(Debug, PartialEq)] +pub(crate) enum GrammarSource { + File(String), + Inline(String), +} + +/// Parsed information of the derive and the attributes. +pub struct ParsedDerive { + /// The identifier of the deriving struct, union, or enum. + pub name: Ident, + /// The generics of the deriving struct, union, or enum. + pub generics: Generics, + /// Indicates whether the 'non_exhaustive' attribute is added to the 'Rule' enum. + pub non_exhaustive: bool, +} + +pub(crate) fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec) { + let name = ast.ident; + let generics = ast.generics; + + let grammar: Vec<&Attribute> = ast + .attrs + .iter() + .filter(|attr| { + let path = attr.meta.path(); + path.is_ident("grammar") || path.is_ident("grammar_inline") + }) + .collect(); + + if grammar.is_empty() { + panic!("a grammar file needs to be provided with the #[grammar = \"PATH\"] or #[grammar_inline = \"GRAMMAR CONTENTS\"] attribute"); + } + + let mut grammar_sources = Vec::with_capacity(grammar.len()); + for attr in grammar { + grammar_sources.push(get_attribute(attr)) + } + + let non_exhaustive = ast + .attrs + .iter() + .any(|attr| attr.meta.path().is_ident("non_exhaustive")); + + ( + ParsedDerive { + name, + generics, + non_exhaustive, + }, + grammar_sources, + ) +} + +fn get_attribute(attr: &Attribute) -> GrammarSource { + match &attr.meta { + Meta::NameValue(name_value) => match &name_value.value { + Expr::Lit(ExprLit { + lit: Lit::Str(string), + .. + }) => { + if name_value.path.is_ident("grammar") { + GrammarSource::File(string.value()) + } else { + GrammarSource::Inline(string.value()) + } + } + _ => panic!("grammar attribute must be a string"), + }, + _ => panic!("grammar attribute must be of the form `grammar = \"...\"`"), + } +} + +#[cfg(test)] +mod tests { + use super::parse_derive; + use super::GrammarSource; + + #[test] + fn derive_inline_file() { + let definition = " + #[other_attr] + #[grammar_inline = \"GRAMMAR\"] + pub struct MyParser<'a, T>; + "; + let ast = syn::parse_str(definition).unwrap(); + let (_, filenames) = parse_derive(ast); + assert_eq!(filenames, [GrammarSource::Inline("GRAMMAR".to_string())]); + } + + #[test] + fn derive_ok() { + let definition = " + #[other_attr] + #[grammar = \"myfile.pest\"] + pub struct MyParser<'a, T>; + "; + let ast = syn::parse_str(definition).unwrap(); + let (parsed_derive, filenames) = parse_derive(ast); + assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); + assert!(!parsed_derive.non_exhaustive); + } + + #[test] + fn derive_multiple_grammars() { + let definition = " + #[other_attr] + #[grammar = \"myfile1.pest\"] + #[grammar = \"myfile2.pest\"] + pub struct MyParser<'a, T>; + "; + let ast = syn::parse_str(definition).unwrap(); + let (_, filenames) = parse_derive(ast); + assert_eq!( + filenames, + [ + GrammarSource::File("myfile1.pest".to_string()), + GrammarSource::File("myfile2.pest".to_string()) + ] + ); + } + + #[test] + fn derive_nonexhaustive() { + let definition = " + #[non_exhaustive] + #[grammar = \"myfile.pest\"] + pub struct MyParser<'a, T>; + "; + let ast = syn::parse_str(definition).unwrap(); + let (parsed_derive, filenames) = parse_derive(ast); + assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); + assert!(parsed_derive.non_exhaustive); + } + + #[test] + #[should_panic(expected = "grammar attribute must be a string")] + fn derive_wrong_arg() { + let definition = " + #[other_attr] + #[grammar = 1] + pub struct MyParser<'a, T>; + "; + let ast = syn::parse_str(definition).unwrap(); + parse_derive(ast); + } + + #[test] + #[should_panic( + expected = "a grammar file needs to be provided with the #[grammar = \"PATH\"] or #[grammar_inline = \"GRAMMAR CONTENTS\"] attribute" + )] + fn derive_no_grammar() { + let definition = " + #[other_attr] + pub struct MyParser<'a, T>; + "; + let ast = syn::parse_str(definition).unwrap(); + parse_derive(ast); + } +}