diff --git a/benches/read_many_structs.rs b/benches/read_many_structs.rs index ddba86f3..d64278a3 100644 --- a/benches/read_many_structs.rs +++ b/benches/read_many_structs.rs @@ -438,7 +438,7 @@ mod benchmark { let mut context = EncodingContext::for_ion_version(IonVersion::v1_1); context .macro_table_mut() - .add_macro(compiled_macro.clone()) + .add_template_macro(compiled_macro.clone()) .unwrap(); let context_ref = context.get_ref(); b.iter(|| { diff --git a/src/lazy/binary/raw/v1_1/immutable_buffer.rs b/src/lazy/binary/raw/v1_1/immutable_buffer.rs index 5f8e2872..d14643fd 100644 --- a/src/lazy/binary/raw/v1_1/immutable_buffer.rs +++ b/src/lazy/binary/raw/v1_1/immutable_buffer.rs @@ -1320,7 +1320,7 @@ mod tests { let mut context = EncodingContext::for_ion_version(IonVersion::v1_1); let template_macro = TemplateCompiler::compile_from_source(context.get_ref(), macro_source)?; - let macro_address = context.macro_table.add_macro(template_macro)?; + let macro_address = context.macro_table.add_template_macro(template_macro)?; let opcode_byte = u8::try_from(macro_address).unwrap(); let binary_ion = encode_macro_fn(opcode_byte as usize); let buffer = BinaryBuffer::new(context.get_ref(), &binary_ion); diff --git a/src/lazy/encoder/text/v1_1/writer.rs b/src/lazy/encoder/text/v1_1/writer.rs index aaf568c0..fa434aeb 100644 --- a/src/lazy/encoder/text/v1_1/writer.rs +++ b/src/lazy/encoder/text/v1_1/writer.rs @@ -286,7 +286,7 @@ mod tests { let mut context = EncodingContext::for_ion_version(IonVersion::v1_1); let macro_foo = TemplateCompiler::compile_from_source(context.get_ref(), "(macro foo (x*) null)")?; - context.macro_table.add_macro(macro_foo)?; + context.macro_table.add_template_macro(macro_foo)?; let context = context.get_ref(); let _marker = reader.next(context)?.expect_ivm()?; let eexp = reader.next(context)?.expect_eexp()?; diff --git a/src/lazy/expanded/e_expression.rs b/src/lazy/expanded/e_expression.rs index 6050479e..44eafdb5 100644 --- a/src/lazy/expanded/e_expression.rs +++ b/src/lazy/expanded/e_expression.rs @@ -9,8 +9,8 @@ use crate::lazy::decoder::{Decoder, RawValueExpr}; use crate::lazy::encoding::TextEncoding_1_1; use crate::lazy::expanded::compiler::{ExpansionAnalysis, ExpansionSingleton}; use crate::lazy::expanded::macro_evaluator::{ - AnnotateExpansion, EExpressionArgGroup, ExprGroupExpansion, IsExhaustedIterator, - MacroExpansion, MacroExpansionKind, MacroExpr, MacroExprArgsIterator, MakeSExpExpansion, + AnnotateExpansion, EExpressionArgGroup, ExprGroupExpansion, FlattenExpansion, + IsExhaustedIterator, MacroExpansion, MacroExpansionKind, MacroExpr, MacroExprArgsIterator, MakeTextExpansion, RawEExpression, TemplateExpansion, ValueExpr, }; use crate::lazy::expanded::macro_table::{MacroKind, MacroRef}; @@ -123,8 +123,12 @@ impl<'top, D: Decoder> EExpression<'top, D> { MacroKind::MakeSymbol => { MacroExpansionKind::MakeSymbol(MakeTextExpansion::symbol_maker(arguments)) } - MacroKind::MakeSExp => MacroExpansionKind::MakeSExp(MakeSExpExpansion::new(arguments)), MacroKind::Annotate => MacroExpansionKind::Annotate(AnnotateExpansion::new(arguments)), + MacroKind::Flatten => MacroExpansionKind::Flatten(FlattenExpansion::new( + self.context, + environment, + arguments, + )), MacroKind::Template(template_body) => { let template_ref = TemplateMacroRef::new(invoked_macro.reference(), template_body); environment = self.new_evaluation_environment()?; diff --git a/src/lazy/expanded/macro_evaluator.rs b/src/lazy/expanded/macro_evaluator.rs index bd270af5..f67bea27 100644 --- a/src/lazy/expanded/macro_evaluator.rs +++ b/src/lazy/expanded/macro_evaluator.rs @@ -13,6 +13,7 @@ #![allow(non_camel_case_types)] use std::fmt::{Debug, Formatter}; +use std::mem; use std::ops::Range; use bumpalo::collections::{String as BumpString, Vec as BumpVec}; @@ -22,7 +23,7 @@ use crate::lazy::expanded::compiler::ExpansionAnalysis; use crate::lazy::expanded::e_expression::{ EExpArgGroup, EExpArgGroupIterator, EExpression, EExpressionArgsIterator, }; -use crate::lazy::expanded::sequence::Environment; +use crate::lazy::expanded::sequence::{Environment, ExpandedSequenceIterator}; use crate::lazy::expanded::template::{ ParameterEncoding, TemplateBodyVariableReference, TemplateExprGroup, TemplateMacroInvocation, TemplateMacroInvocationArgsIterator, TemplateMacroRef, @@ -34,8 +35,8 @@ use crate::lazy::text::raw::v1_1::arg_group::EExpArg; use crate::lazy::text::raw::v1_1::reader::MacroIdRef; use crate::result::IonFailure; use crate::{ - ExpandedSExpSource, ExpandedValueSource, IonError, IonResult, LazyExpandedSExp, LazySExp, - LazyValue, Span, SymbolRef, ValueRef, + ExpandedValueRef, ExpandedValueSource, IonError, IonResult, LazyValue, Span, SymbolRef, + ValueRef, }; pub trait IsExhaustedIterator<'top, D: Decoder>: @@ -419,20 +420,19 @@ impl<'top, D: Decoder> ValueExpr<'top, D> { /// Indicates which of the supported macros this represents and stores the state necessary to /// continue evaluating that macro. -#[derive(Copy, Clone, Debug)] +#[derive(Debug)] pub enum MacroExpansionKind<'top, D: Decoder> { None, // `(.none)` returns the empty stream ExprGroup(ExprGroupExpansion<'top, D>), MakeString(MakeTextExpansion<'top, D>), MakeSymbol(MakeTextExpansion<'top, D>), - MakeSExp(MakeSExpExpansion<'top, D>), Annotate(AnnotateExpansion<'top, D>), + Flatten(FlattenExpansion<'top, D>), Template(TemplateExpansion<'top>), } /// A macro in the process of being evaluated. Stores both the state of the evaluation and the /// syntactic element that represented the macro invocation. -#[derive(Copy, Clone)] pub struct MacroExpansion<'top, D: Decoder> { context: EncodingContextRef<'top>, kind: MacroExpansionKind<'top, D>, @@ -499,18 +499,12 @@ impl<'top, D: Decoder> MacroExpansion<'top, D> { ExprGroup(expr_group_expansion) => expr_group_expansion.next(context, environment), MakeString(make_string_expansion) => make_string_expansion.make_text_value(context), MakeSymbol(make_symbol_expansion) => make_symbol_expansion.make_text_value(context), - MakeSExp(make_sexp_expansion) => make_sexp_expansion.next(context, environment), Annotate(annotate_expansion) => annotate_expansion.next(context, environment), + Flatten(flatten_expansion) => flatten_expansion.next(), // `none` is trivial and requires no delegation None => Ok(MacroExpansionStep::FinalStep(Option::None)), } } - - // Calculate the next step in this macro expansion without advancing the expansion. - pub fn peek_next_step(&self) -> IonResult> { - let mut expansion_copy = *self; - expansion_copy.next_step() - } } impl Debug for MacroExpansion<'_, D> { @@ -520,8 +514,8 @@ impl Debug for MacroExpansion<'_, D> { MacroExpansionKind::ExprGroup(_) => "[internal] expr_group", MacroExpansionKind::MakeString(_) => "make_string", MacroExpansionKind::MakeSymbol(_) => "make_symbol", - MacroExpansionKind::MakeSExp(_) => "make_sexp", MacroExpansionKind::Annotate(_) => "annotate", + MacroExpansionKind::Flatten(_) => "flatten", MacroExpansionKind::Template(t) => { return if let Some(name) = t.template.name() { write!(f, "", name) @@ -718,7 +712,7 @@ impl<'top, D: Decoder> MacroEvaluator<'top, D> { #[inline(never)] pub fn push_general_case(&mut self, new_expansion: MacroExpansion<'top, D>) { - match self.state { + match mem::take(&mut self.state) { // Going from zero expansions to one expansion EvaluatorState::Empty => self.state = EvaluatorState::Stackless(new_expansion), // Going from one expansion to two @@ -729,7 +723,7 @@ impl<'top, D: Decoder> MacroEvaluator<'top, D> { ); stacked_evaluator .macro_stack - .extend_from_slice_copy(&[original_expansion, new_expansion]); + .extend([original_expansion, new_expansion]); self.state = EvaluatorState::Stacked(stacked_evaluator) } // Going from 2+ up @@ -1070,40 +1064,99 @@ impl<'top, D: Decoder> MakeTextExpansion<'top, D> { } } -// ====== Implementation of the `make_sexp` macro +// ====== Implementation of the `flatten` macro -#[derive(Copy, Clone, Debug)] -pub struct MakeSExpExpansion<'top, D: Decoder> { +#[derive(Debug)] +pub struct FlattenExpansion<'top, D: Decoder> { arguments: MacroExprArgsIterator<'top, D>, + evaluator: &'top mut MacroEvaluator<'top, D>, + // This is &mut Option<_> instead of Option<&mut _> because it allows us to do a single + // bump-allocation up front and re-use that space to hold each of the iterators we'll + // work with over the course of evaluation. + // In plainer terms: we _always_ have an allocated space that may or may not contain an iterator. + // We can put iterators into that space or remove them. + // If this were Option<&mut _>, we _might_ have a space with an iterator in it. If we set a new + // iterator, we would have to allocate a space for that iterator. + current_sequence: &'top mut Option>, } -impl<'top, D: Decoder> MakeSExpExpansion<'top, D> { - pub fn new(arguments: MacroExprArgsIterator<'top, D>) -> Self { - Self { arguments } - } - - /// Yields the next [`ValueExpr`] in this `make_sexp` macro's evaluation. - pub fn next( - &mut self, +impl<'top, D: Decoder> FlattenExpansion<'top, D> { + pub fn new( context: EncodingContextRef<'top>, environment: Environment<'top, D>, - ) -> IonResult> { - // The `make_sexp` macro always produces a single s-expression. When `next()` is called - // to begin its evaluation, immediately return a lazy value representing the (not yet - // computed) sexp. If/when the application tries to iterate over its child expressions, - // the iterator will evaluate the child expressions incrementally. - let lazy_expanded_sexp = LazyExpandedSExp { - source: ExpandedSExpSource::Constructed(environment, self.arguments), - context, + arguments: MacroExprArgsIterator<'top, D>, + ) -> Self { + let allocator = context.allocator(); + let evaluator = allocator.alloc_with(|| MacroEvaluator::new_with_environment(environment)); + let current_sequence = allocator.alloc_with(|| None); + Self { + evaluator, + arguments, + current_sequence, + } + } + + fn set_current_sequence(&mut self, value: LazyExpandedValue<'top, D>) -> IonResult<()> { + *self.current_sequence = match value.read()? { + ExpandedValueRef::List(list) => Some(ExpandedSequenceIterator::List(list.iter())), + ExpandedValueRef::SExp(sexp) => Some(ExpandedSequenceIterator::SExp(sexp.iter())), + other => { + return IonResult::decoding_error(format!( + "`flatten` only accepts sequences, received {other:?}" + )) + } }; - let lazy_sexp = LazySExp::new(lazy_expanded_sexp); - // Store the `SExp` in the bump so it's guaranteed to be around as long as the reader is - // positioned on this top-level value. - let value_ref = context.allocator().alloc_with(|| ValueRef::SExp(lazy_sexp)); - let lazy_expanded_value = LazyExpandedValue::from_constructed(context, &[], value_ref); - Ok(MacroExpansionStep::FinalStep(Some( - ValueExpr::ValueLiteral(lazy_expanded_value), - ))) + Ok(()) + } + + /// Yields the next [`ValueExpr`] in this `flatten` macro's evaluation. + fn next(&mut self) -> IonResult> { + loop { + // If we're already flattening a sequence, get the next nested value from it. + if let Some(current_sequence) = self.current_sequence { + // First, get the next nested sequence value result from the iterator. + match current_sequence.next() { + // If we get `Some(IonResult)`, return it even if it's an Err. + Some(Ok(result)) => { + return Ok(MacroExpansionStep::Step(ValueExpr::ValueLiteral(result))) + } + Some(Err(e)) => return Err(e), + // If we get `None`, the iterator is exhausted and we should continue on to the next sequence. + None => *self.current_sequence = None, + } + } + + // If we reach this point, we don't have a current sequence. + // We've either just started evaluation and haven't set one yet or + // we just finished flattening a sequence and need to set a new one. + + // See if the evaluator has an expansion in progress. + let mut next_seq = self.evaluator.next()?; + + if next_seq.is_none() { + // If we don't get anything from the evaluator, we'll get our sequence from the + // next argument expression. + next_seq = match self.arguments.next().transpose()? { + // If the expression is a value literal, that's our new sequence. + Some(ValueExpr::ValueLiteral(value)) => Some(value), + // If the expression is a macro invocation, we'll start evaluating it + // and return to the top of the loop. + Some(ValueExpr::MacroInvocation(invocation)) => { + self.evaluator.push(invocation.expand()?); + continue; + } + // If there isn't a next argument expression, then evaluation is complete. + None => return Ok(MacroExpansionStep::FinalStep(None)), + } + } + + // At this point, `next_seq` is definitely populated, so we can safely unwrap it. + let next_seq = next_seq.unwrap(); + + // Set it as our new current sequence. This step also type-checks the value to confirm + // that it is either a list or an s-expression. + self.set_current_sequence(next_seq)?; + } } } @@ -2434,6 +2487,23 @@ mod tests { ) } + #[test] + fn flatten_e_expression() -> IonResult<()> { + stream_eq( + r#" + (:flatten + [1, 2, 3] + [] + [] + (4 5 6) + () + () + (7)) + "#, + r#" 1 2 3 4 5 6 7 "#, + ) + } + #[test] fn make_sexp_e_expression() -> IonResult<()> { let e_expression = r#" @@ -2449,6 +2519,36 @@ mod tests { stream_eq(e_expression, r#" (1 2 3 4 5 6 7) "#) } + #[test] + fn make_list_e_expression() -> IonResult<()> { + let e_expression = r#" + (:make_list + [1, 2, 3] + [] + [] + (4 5 6) + () + () + (7)) + "#; + stream_eq(e_expression, r#" [1, 2, 3, 4, 5, 6, 7] "#) + } + + #[test] + fn make_list_with_nested_eexp() -> IonResult<()> { + let e_expression = r#" + (:make_list + [1, 2, 3] + [] + [] + ((:values 4 (:values 5 6))) + () + () + (7)) + "#; + stream_eq(e_expression, r#" [1, 2, 3, 4, 5, 6, 7] "#) + } + #[test] fn make_string_tdl_macro_invocation() -> IonResult<()> { let invocation = r#" diff --git a/src/lazy/expanded/macro_table.rs b/src/lazy/expanded/macro_table.rs index 4885dc89..25f9ddae 100644 --- a/src/lazy/expanded/macro_table.rs +++ b/src/lazy/expanded/macro_table.rs @@ -8,6 +8,7 @@ use crate::{EncodingContext, IonResult, IonType, IonVersion, TemplateCompiler}; use compact_str::CompactString; use delegate::delegate; use rustc_hash::{FxBuildHasher, FxHashMap}; +use std::cell::RefCell; use std::sync::{Arc, LazyLock}; #[derive(Debug, Clone, PartialEq)] @@ -113,8 +114,8 @@ pub enum MacroKind { ExprGroup, MakeString, MakeSymbol, - MakeSExp, Annotate, + Flatten, Template(TemplateBody), // A placeholder for not-yet-implemented macros ToDo, @@ -188,13 +189,6 @@ impl Default for MacroTable { } impl MacroTable { - pub const SYSTEM_MACRO_KINDS: &'static [MacroKind] = &[ - MacroKind::None, - MacroKind::ExprGroup, - MacroKind::MakeString, - MacroKind::MakeSExp, - MacroKind::Annotate, - ]; // The system macros range from address 0 to 23 pub const NUM_SYSTEM_MACROS: usize = 24; // When a user defines new macros, this is the first ID that will be assigned. This value @@ -202,14 +196,25 @@ impl MacroTable { pub const FIRST_USER_MACRO_ID: usize = Self::NUM_SYSTEM_MACROS; fn compile_system_macros() -> Vec> { - let bootstrap_context = EncodingContext::empty(); - let context = bootstrap_context.get_ref(); + // This is wrapped in a `RefCell` in order to allow two different closures to hold + // runtime-checked mutating references to the context. This overhead is minimal and is only + // paid during the initialization of the singleton system macro table. + let bootstrap_context = RefCell::new(EncodingContext::empty()); // Creates a `Macro` from a TDL expression let template = |source: &str| { - Arc::new(Macro::from_template_macro( - TemplateCompiler::compile_from_source(context, source).unwrap(), - )) + // Compile the given TDL source expression using the current context. + let macro_ref = Arc::new(Macro::from_template_macro( + TemplateCompiler::compile_from_source(bootstrap_context.borrow().get_ref(), source) + .unwrap(), + )); + // Add the new macro to the context so 'downstream' macros can invoke it. + bootstrap_context + .borrow_mut() + .macro_table_mut() + .append_macro(¯o_ref) + .unwrap(); + macro_ref }; // Creates a `Macro` whose implementation is provided by the system @@ -217,14 +222,36 @@ impl MacroTable { signature: &str, kind: MacroKind, expansion_analysis: ExpansionAnalysis| { - Arc::new(Macro::named( + // Construct a macro from the provided parameters using the current context. + let macro_ref = Arc::new(Macro::named( name, - TemplateCompiler::compile_signature(context, signature).unwrap(), + TemplateCompiler::compile_signature( + bootstrap_context.borrow().get_ref(), + signature, + ) + .unwrap(), kind, expansion_analysis, - )) + )); + // Add the new macro to the context so 'downstream' macros can invoke it. + bootstrap_context + .borrow_mut() + .macro_table_mut() + .append_macro(¯o_ref) + .unwrap(); + macro_ref }; + // `make_sexp` and `make_list` depend on `flatten`, which happens to be defined later in the + // table. We define it in advance so it will already be in the context when `make_sexp` and + // `make_list` are defined. + let flatten_macro_definition = builtin( + "flatten", + "(sequences*)", + MacroKind::Flatten, + ExpansionAnalysis::application_value_stream(), + ); + // Macro definitions in the system table are encoded in **Ion 1.0** because it does not // require the Ion 1.1 system macros to exist. vec![ @@ -271,17 +298,17 @@ impl MacroTable { MacroKind::ToDo, ExpansionAnalysis::single_application_value(IonType::Timestamp), ), - builtin( - "make_list", - "(sequences*)", - MacroKind::ToDo, - ExpansionAnalysis::single_application_value(IonType::List), + template( + r#" + (macro make_list (sequences*) + [(.flatten (%sequences))]) + "#, ), - builtin( - "make_sexp", - "(sequences*)", - MacroKind::MakeSExp, - ExpansionAnalysis::single_application_value(IonType::SExp), + template( + r#" + (macro make_sexp (sequences*) + ((.flatten (%sequences)))) + "#, ), builtin( "make_struct", @@ -365,24 +392,14 @@ impl MacroTable { MacroKind::ToDo, ExpansionAnalysis::application_value_stream(), ), - builtin( - "flatten", - "(sequences*)", - MacroKind::ToDo, - ExpansionAnalysis::application_value_stream(), - ), + flatten_macro_definition, builtin( "sum", "(a b)", MacroKind::ToDo, ExpansionAnalysis::single_application_value(IonType::Int), ), - builtin( - "meta", - "(expr*)", - MacroKind::ToDo, - ExpansionAnalysis::no_assertions_made(), - ), + template("(macro meta (expr*) (.none))"), builtin( "make_field", "(name value)", @@ -490,7 +507,7 @@ impl MacroTable { } } - pub fn add_macro(&mut self, template: TemplateMacro) -> IonResult { + pub fn add_template_macro(&mut self, template: TemplateMacro) -> IonResult { let id = self.macros_by_address.len(); // If the macro has a name, make sure that name is not already in use and then add it. if let Some(name) = &template.name { @@ -511,18 +528,21 @@ impl MacroTable { Ok(id) } + pub(crate) fn append_macro(&mut self, macro_ref: &Arc) -> IonResult<()> { + let next_id = self.len(); + if let Some(name) = macro_ref.clone_name() { + if self.macros_by_name.contains_key(name.as_str()) { + return IonResult::decoding_error(format!("macro named '{name}' already exists")); + } + self.macros_by_name.insert(name, next_id); + } + self.macros_by_address.push(Arc::clone(macro_ref)); + Ok(()) + } + pub(crate) fn append_all_macros_from(&mut self, other: &MacroTable) -> IonResult<()> { for macro_ref in &other.macros_by_address { - let next_id = self.len(); - if let Some(name) = macro_ref.clone_name() { - if self.macros_by_name.contains_key(name.as_str()) { - return IonResult::decoding_error(format!( - "macro named '{name}' already exists" - )); - } - self.macros_by_name.insert(name, next_id); - } - self.macros_by_address.push(Arc::clone(macro_ref)) + self.append_macro(macro_ref)? } Ok(()) } diff --git a/src/lazy/expanded/mod.rs b/src/lazy/expanded/mod.rs index 43ee74fd..9c1f1a1d 100644 --- a/src/lazy/expanded/mod.rs +++ b/src/lazy/expanded/mod.rs @@ -150,7 +150,7 @@ impl EncodingContext { } pub fn register_template(&mut self, template_macro: TemplateMacro) -> IonResult { - self.macro_table.add_macro(template_macro) + self.macro_table.add_template_macro(template_macro) } } @@ -288,7 +288,7 @@ impl ExpandingReader { fn add_macro(&mut self, template_macro: TemplateMacro) -> IonResult { let macro_table = &mut self.context_mut().macro_table; - macro_table.add_macro(template_macro) + macro_table.add_template_macro(template_macro) } pub fn context(&self) -> EncodingContextRef<'_> { diff --git a/src/lazy/expanded/sequence.rs b/src/lazy/expanded/sequence.rs index 7370b5c0..5ea01ff6 100644 --- a/src/lazy/expanded/sequence.rs +++ b/src/lazy/expanded/sequence.rs @@ -1,14 +1,11 @@ use crate::element::iterators::SymbolsIterator; use crate::lazy::decoder::{Decoder, LazyRawSequence, LazyRawValueExpr}; -use crate::lazy::expanded::macro_evaluator::{ - MacroEvaluator, MacroExprArgsIterator, RawEExpression, ValueExpr, -}; +use crate::lazy::expanded::macro_evaluator::{MacroEvaluator, RawEExpression, ValueExpr}; use crate::lazy::expanded::template::{TemplateElement, TemplateSequenceIterator}; use crate::lazy::expanded::{ EncodingContextRef, ExpandedAnnotationsIterator, ExpandedAnnotationsSource, LazyExpandedValue, }; -use crate::result::IonFailure; -use crate::{try_or_some_err, ExpandedValueRef, IonResult, IonType, SymbolRef}; +use crate::{try_or_some_err, IonResult, IonType}; use std::fmt::Debug; /// A sequence of not-yet-evaluated expressions passed as arguments to a macro invocation. @@ -173,6 +170,7 @@ impl<'top, D: Decoder> LazyExpandedList<'top, D> { } /// The source of child values iterated over by an [`ExpandedListIterator`]. +#[derive(Debug)] pub enum ExpandedListIteratorSource<'top, D: Decoder> { ValueLiteral( // Giving the list iterator its own evaluator means that we can abandon the iterator @@ -185,6 +183,7 @@ pub enum ExpandedListIteratorSource<'top, D: Decoder> { } /// Iterates over the child values of a [`LazyExpandedList`]. +#[derive(Debug)] pub struct ExpandedListIterator<'top, D: Decoder> { context: EncodingContextRef<'top>, source: ExpandedListIteratorSource<'top, D>, @@ -212,8 +211,6 @@ pub enum ExpandedSExpSource<'top, D: Decoder> { /// Because the TDL treats s-expressions as literals, this variant only applies when the /// s-expression appeared within a `literal` operation. Template(Environment<'top, D>, TemplateElement<'top>), - /// The SExp was produced by a call to `make_sexp`. - Constructed(Environment<'top, D>, MacroExprArgsIterator<'top, D>), } /// An s-expression that may have come from either a value literal in the input stream or from @@ -244,13 +241,6 @@ impl<'top, D: Decoder> LazyExpandedSExp<'top, D> { source: ExpandedAnnotationsSource::Template(SymbolsIterator::new(annotations)), } } - ExpandedSExpSource::Constructed(_environment, _args) => { - // `make_sexp` always produces an unannotated s-expression - const EMPTY_ANNOTATIONS: &[SymbolRef<'_>] = &[]; - ExpandedAnnotationsIterator { - source: ExpandedAnnotationsSource::Constructed(EMPTY_ANNOTATIONS.iter()), - } - } } } @@ -269,10 +259,6 @@ impl<'top, D: Decoder> LazyExpandedSExp<'top, D> { nested_expressions, )) } - ExpandedSExpSource::Constructed(environment, args) => { - let evaluator = MacroEvaluator::new_with_environment(*environment); - ExpandedSExpIteratorSource::Constructed(evaluator, *args, None) - } }; ExpandedSExpIterator { context: self.context, @@ -299,6 +285,7 @@ impl<'top, D: Decoder> LazyExpandedSExp<'top, D> { } /// The source of child values iterated over by an [`ExpandedSExpIterator`]. +#[derive(Debug)] pub enum ExpandedSExpIteratorSource<'top, D: Decoder> { /// The SExp was a literal in the data stream ValueLiteral( @@ -309,17 +296,10 @@ pub enum ExpandedSExpIteratorSource<'top, D: Decoder> { ), /// The SExp was in the body of a macro Template(TemplateSequenceIterator<'top, D>), - /// The SExp came from a call to `make_sexp` - Constructed( - MacroEvaluator<'top, D>, - // The argument expressions passed to `make_sexp` - MacroExprArgsIterator<'top, D>, - // The list or sexp whose contents are currently being traversed by the iterator - Option>, - ), } /// Iterates over the child values of a [`LazyExpandedSExp`]. +#[derive(Debug)] pub struct ExpandedSExpIterator<'top, D: Decoder> { context: EncodingContextRef<'top>, source: ExpandedSExpIteratorSource<'top, D>, @@ -335,56 +315,6 @@ impl<'top, D: Decoder> Iterator for ExpandedSExpIterator<'top, D> { expand_next_sequence_value(self.context, evaluator, iter) } Template(iter) => iter.next(), - Constructed(evaluator, iter, current_sequence) => { - Self::next_concatenated_value(evaluator, iter, current_sequence) - } - } - } -} - -impl<'top, D: Decoder> ExpandedSExpIterator<'top, D> { - fn next_concatenated_value( - evaluator: &mut MacroEvaluator<'top, D>, - iter: &mut MacroExprArgsIterator<'top, D>, - current_sequence: &mut Option>, - ) -> Option>> { - loop { - // If we're currently traversing a list or sexp, get the next value out of it. - if let Some(sequence) = current_sequence { - if let Some(result) = sequence.next() { - return Some(result); - } - } - - // If we get this far, any sequence we may have been traversing is now exhausted. We should - // start evaluating the next expression from `iter`, expecting another sequence to traverse. - let value = match expand_next_sequence_value_from_resolved(evaluator, iter) { - Some(Ok(value)) => value, - Some(Err(e)) => return Some(Err(e)), - None => return None, - }; - - // We got another value from our iterator. Make sure it's a sequence and then store - // an iterator for it - match try_or_some_err!(value.read()) { - ExpandedValueRef::List(list) => { - *current_sequence = { - let list_iter = value.context().allocator().alloc_with(|| list.iter()); - Some(FlattenedSequence::List(list_iter)) - } - } - ExpandedValueRef::SExp(sexp) => { - *current_sequence = { - let sexp_iter = value.context().allocator().alloc_with(|| sexp.iter()); - Some(FlattenedSequence::SExp(sexp_iter)) - } - } - other => { - return Some(IonResult::decoding_error(format!( - "`make_sexp` arguments must be sequences (list or sexp); found {other:?}" - ))) - } - } } } } @@ -430,18 +360,19 @@ fn expand_next_sequence_value_from_resolved<'top, D: Decoder>( } /// Represents a sequence (list or sexp) whose contents are being traversed. -/// This is used in (e.g.) `make_sexp` to store an iterator for each of its +/// This is used in `flatten` to store an iterator for each of its /// sequence arguments in turn. -pub enum FlattenedSequence<'top, D: Decoder> { - List(&'top mut ExpandedListIterator<'top, D>), - SExp(&'top mut ExpandedSExpIterator<'top, D>), +#[derive(Debug)] +pub enum ExpandedSequenceIterator<'top, D: Decoder> { + List(ExpandedListIterator<'top, D>), + SExp(ExpandedSExpIterator<'top, D>), } -impl<'top, D: Decoder> Iterator for FlattenedSequence<'top, D> { +impl<'top, D: Decoder> Iterator for ExpandedSequenceIterator<'top, D> { type Item = IonResult>; fn next(&mut self) -> Option { - use FlattenedSequence::*; + use ExpandedSequenceIterator::*; match self { List(l) => l.next(), SExp(s) => s.next(), diff --git a/src/lazy/expanded/template.rs b/src/lazy/expanded/template.rs index 7b3ead81..fae796de 100644 --- a/src/lazy/expanded/template.rs +++ b/src/lazy/expanded/template.rs @@ -8,7 +8,7 @@ use compact_str::CompactString; use crate::lazy::binary::raw::v1_1::immutable_buffer::ArgGroupingBitmap; use crate::lazy::decoder::Decoder; use crate::lazy::expanded::compiler::ExpansionAnalysis; -use crate::lazy::expanded::macro_evaluator::{AnnotateExpansion, MacroEvaluator, MacroExpansion, MacroExpansionKind, MacroExpr, MacroExprArgsIterator, MakeSExpExpansion, TemplateExpansion, ValueExpr, ExprGroupExpansion, MakeTextExpansion}; +use crate::lazy::expanded::macro_evaluator::{AnnotateExpansion, MacroEvaluator, MacroExpansion, MacroExpansionKind, MacroExpr, MacroExprArgsIterator, TemplateExpansion, ValueExpr, ExprGroupExpansion, MakeTextExpansion, FlattenExpansion}; use crate::lazy::expanded::macro_table::{Macro, MacroKind}; use crate::lazy::expanded::r#struct::UnexpandedField; use crate::lazy::expanded::sequence::Environment; @@ -435,6 +435,7 @@ impl<'top> Deref for TemplateMacroRef<'top> { } /// Steps over the child expressions of a list or s-expression found in the body of a template. +#[derive(Debug)] pub struct TemplateSequenceIterator<'top, D: Decoder> { context: EncodingContextRef<'top>, template: TemplateMacroRef<'top>, @@ -1220,8 +1221,8 @@ impl<'top, D: Decoder> TemplateMacroInvocation<'top, D> { MacroKind::MakeSymbol => { MacroExpansionKind::MakeSymbol(MakeTextExpansion::symbol_maker(arguments)) } - MacroKind::MakeSExp => MacroExpansionKind::MakeSExp(MakeSExpExpansion::new(arguments)), MacroKind::Annotate => MacroExpansionKind::Annotate(AnnotateExpansion::new(arguments)), + MacroKind::Flatten => MacroExpansionKind::Flatten(FlattenExpansion::new(self.context(), self.environment, arguments)), MacroKind::Template(template_body) => { let template_ref = TemplateMacroRef::new(macro_ref, template_body); let new_environment = self.new_evaluation_environment()?; diff --git a/src/lazy/sequence.rs b/src/lazy/sequence.rs index 380689d8..96ce53e8 100644 --- a/src/lazy/sequence.rs +++ b/src/lazy/sequence.rs @@ -9,7 +9,7 @@ use crate::lazy::expanded::sequence::{ use crate::lazy::value::{AnnotationsIterator, LazyValue}; use crate::{ try_next, Annotations, Element, ExpandedListSource, ExpandedSExpSource, IntoAnnotatedElement, - LazyExpandedValue, LazyRawContainer, Sequence, Value, ValueRef, + LazyExpandedValue, LazyRawContainer, Sequence, Value, }; use crate::{IonError, IonResult}; @@ -245,11 +245,6 @@ impl<'top, D: Decoder> LazySExp<'top, D> { ExpandedSExpSource::Template(env, element) => { LazyExpandedValue::from_template(context, env, element) } - ExpandedSExpSource::Constructed(_environment, _args) => { - let value_ref = context.allocator().alloc_with(|| ValueRef::SExp(*self)); - let annotations = &[]; - LazyExpandedValue::from_constructed(context, annotations, value_ref) - } }; LazyValue::new(expanded_value) } diff --git a/src/lazy/system_reader.rs b/src/lazy/system_reader.rs index b2e000df..ebbba7de 100644 --- a/src/lazy/system_reader.rs +++ b/src/lazy/system_reader.rs @@ -413,7 +413,7 @@ impl SystemReader { ValueRef::SExp(macro_def_sexp) => { let new_macro = TemplateCompiler::compile_from_sexp(context, ¯o_table, macro_def_sexp)?; - macro_table.add_macro(new_macro)?; + macro_table.add_template_macro(new_macro)?; } ValueRef::Symbol(module_name) if module_name == v1_1::constants::DEFAULT_MODULE_NAME => @@ -686,8 +686,7 @@ mod tests { use crate::lazy::decoder::RawVersionMarker; use crate::lazy::system_stream_item::SystemStreamItem; use crate::{ - constants, v1_0, v1_1, AnyEncoding, Catalog, IonResult, SequenceWriter, SymbolRef, - ValueWriter, Writer, + v1_0, AnyEncoding, Catalog, IonResult, SequenceWriter, SymbolRef, ValueWriter, Writer, }; use super::*; diff --git a/src/lazy/text/buffer.rs b/src/lazy/text/buffer.rs index 08438e45..ee02ca46 100644 --- a/src/lazy/text/buffer.rs +++ b/src/lazy/text/buffer.rs @@ -2822,7 +2822,10 @@ mod tests { fn register_macro(&mut self, text: &str) -> &mut Self { let new_macro = TemplateCompiler::compile_from_source(self.context.get_ref(), text).unwrap(); - self.context.macro_table.add_macro(new_macro).unwrap(); + self.context + .macro_table + .add_template_macro(new_macro) + .unwrap(); self } diff --git a/src/lazy/text/raw/v1_1/reader.rs b/src/lazy/text/raw/v1_1/reader.rs index 7d57f949..ba899f78 100644 --- a/src/lazy/text/raw/v1_1/reader.rs +++ b/src/lazy/text/raw/v1_1/reader.rs @@ -874,7 +874,7 @@ mod tests { let mut context = EncodingContext::for_ion_version(IonVersion::v1_1); let macro_quux = TemplateCompiler::compile_from_source(context.get_ref(), "(macro quux (x) null)")?; - context.macro_table.add_macro(macro_quux)?; + context.macro_table.add_template_macro(macro_quux)?; let reader = &mut LazyRawTextReader_1_1::new(data.as_bytes()); let context = context.get_ref();