Add conformance testing DSL (#826)

* Add conformance testing DSL implementation, CI/CD runner, and user targettable test

* Fix remaining paths for ion 1.0 iontestdata

* Handle typed nulls in read_resolved

* Add support for more data model clauses

* Remove left over comments; fix byte range; support multiple string in text clause

* Update 1.1 skip list to address ion-test path changes

* Address clippy checks, remove unused test

* Drop path separators from path replacement so the canonicalized windows version can match

* Remove conformance cli from default tests

* Add encoding and mactab fragments

* Add timestamp to denote data model

* Missed some uses when --patch'ing

* Address PR feedback; Remove unused Each variant of Fragment since it is not a fragment

Cargo.toml

@@ -94,3 +94,8 @@ codegen-units = 1
 [profile.profiling]
 inherits = "release"
 debug = true
+
+[[test]]
+name = "conformance"
+harness = false
+test = false

src/lazy/binary/raw/v1_1/value.rs

@@ -119,7 +119,13 @@ impl<'top> HasRange for &'top LazyRawBinaryValue_1_1<'top> {
 
 impl<'top> LazyRawValue<'top, BinaryEncoding_1_1> for &'top LazyRawBinaryValue_1_1<'top> {
     fn ion_type(&self) -> IonType {
-        self.encoded_value.ion_type()
+        // Handle retrieving the type for a typed null.
+        if self.encoded_value.header.type_code() == OpcodeType::TypedNull {
+            let body = self.value_body();
+            ION_1_1_TYPED_NULL_TYPES[body[0] as usize]
+        } else {
+            self.encoded_value.ion_type()
+        }
     }
 
     fn is_null(&self) -> bool {
@@ -145,13 +151,7 @@ impl<'top> LazyRawValue<'top, BinaryEncoding_1_1> for &'top LazyRawBinaryValue_1
         }
 
         if self.is_null() {
-            let ion_type = if self.encoded_value.header.ion_type_code == OpcodeType::TypedNull {
-                let body = self.value_body();
-                ION_1_1_TYPED_NULL_TYPES[body[0] as usize]
-            } else {
-                IonType::Null
-            };
-            return Ok(RawValueRef::Null(ion_type));
+            return Ok(RawValueRef::Null(self.ion_type()));
         }
 
         match self.ion_type() {

tests/conformance.rs

@@ -0,0 +1,54 @@
+
+#[cfg(feature = "experimental-reader-writer")]
+mod conformance_dsl;
+
+#[cfg(feature = "experimental-reader-writer")]
+pub fn main() {
+    use crate::conformance_dsl::prelude::*;
+
+    let test_paths = std::env::args().skip(1).collect::<Vec<String>>();
+    let mut errors: Vec<(String, String, conformance_dsl::ConformanceError)> = vec!();
+
+    println!("Testing {} conformance collections.\n", test_paths.len());
+
+    let mut failures = 0;
+
+    for test_path in test_paths {
+        println!("\nRunning tests: {} ========================", test_path);
+        let collection = TestCollection::load(&test_path).expect("unable to load test file");
+        let name_len = collection.iter().fold(0, |acc, d| std::cmp::max(acc, d.name.as_ref().map_or(0, |n| n.len())));
+
+        for doc in collection.iter() {
+            match doc.name.as_ref() {
+                Some(n) => print!("   {:<width$}", n, width = name_len),
+                None => print!("   {:<width$}", "<unnamed>", width = name_len),
+            }
+
+            print!("  ...  ");
+            match doc.run() {
+                Err(e) => {
+                    println!("[FAILED]");
+                    failures += 1;
+                    errors.push((test_path.to_owned(), doc.name.as_deref().unwrap_or("<unnamed>").to_owned(), e.clone()));
+                }
+                Ok(_) => println!("[OK]"),
+            }
+        }
+    }
+
+    for (test_path, test_name, err) in errors {
+        println!("-------------------------");
+        println!("File: {}", test_path);
+        println!("Test: {}", test_name);
+        println!("Error: {:?}", err);
+    }
+
+    if failures > 0 {
+        panic!("Conformance test(s) failed");
+    }
+}
+
+#[cfg(not(feature = "experimental-reader-writer"))]
+pub fn main() {
+    println!("Needs feature experimental-reader-writer");
+}

tests/conformance_dsl/clause.rs

@@ -0,0 +1,160 @@
+//! A Clause represents the DSL's S-Expression operations for defining tests. Each possible
+//! expression should come from a Clause.
+//!
+//! The grammar defining each of the clauses can be found [here][Grammar].
+//!
+//! [Grammar]: https://github.com/amazon-ion/ion-tests/blob/master/conformance/README.md#grammar
+
+use std::str::FromStr;
+
+use ion_rs::{Element, Sequence};
+
+use super::*;
+
+/// Represents each type of S-Expression Clause that we can have in the DSL. This currently does
+/// not capture the Data Model clauses used in Denotes fragments.
+#[allow(non_camel_case_types)]
+#[derive(Debug)]
+pub(crate) enum ClauseType {
+    /// Start an ion 1.0 test document.
+    Ion1_0,
+    /// Start an ion 1.1 test document.
+    Ion1_1,
+    /// Start a test document that validates both ion 1.1 and 1.0
+    Ion1_X,
+    /// Provide a string as text ion, that will be inserted into the test document.
+    Text,
+    /// Provide a sequence of bytes that is interpreted as binary ion, that will be inserted into
+    /// the document.
+    Binary,
+    /// Provide a major and minor version that will be emitted into the document as an IVM.
+    Ivm,
+    /// Specify a ion data to be inserted into the document, using inline ion syntax.
+    TopLevel,
+    /// Provide ion data defining the contents of an '$ion_encoding' directive.
+    Encoding,
+    /// Provide ion data defining the contents of a macro table wrapped by a module within an encoding directive.
+    MacTab,
+    /// Define data that is expected to be produced by the test's document, using inline ion
+    /// syntax.
+    Produces,
+    /// Define data that is expected to be produced by the test's document, using a clause-based
+    /// data model.
+    Denotes,
+    /// Specify that the test should signal (fail).
+    Signals,
+    /// Evaluate the logical conjunction of the clause's arguments.
+    And,
+    /// Negate the evaluation of the clause's argument.
+    Not,
+    /// A continuation that allows for the chaining of fragments and expectations.
+    Then,
+    /// Specify the start of a test.
+    Document,
+    /// Combine one or more continuations with a parent document separately.
+    Each,
+    /// Define a symbol using both text and symbol id for testing in a denotes clause.
+    Absent,
+}
+
+impl FromStr for ClauseType {
+    type Err = ConformanceErrorKind;
+
+    fn from_str(s: &str) -> InnerResult<Self> {
+        use ClauseType::*;
+
+        match s {
+            "ion_1_0" => Ok(Ion1_0),
+            "ion_1_1" => Ok(Ion1_1),
+            "ion_1_x" => Ok(Ion1_X),
+            "document" => Ok(Document),
+            "toplevel" => Ok(TopLevel),
+            "produces" => Ok(Produces),
+            "denotes" => Ok(Denotes),
+            "text" => Ok(Text),
+            "binary" => Ok(Binary),
+            "and" => Ok(And),
+            "not" => Ok(Not),
+            "then" => Ok(Then),
+            "each" => Ok(Each),
+            "absent" => Ok(Absent),
+            "ivm" => Ok(Ivm),
+            "signals" => Ok(Signals),
+            "encoding" => Ok(Encoding),
+            "mactab" => Ok(MacTab),
+            _ => Err(ConformanceErrorKind::UnknownClause(s.to_owned())),
+        }
+    }
+}
+
+impl ClauseType {
+
+    /// Utility function to test if the Clause is a fragment node.
+    pub fn is_fragment(&self) -> bool {
+        use ClauseType::*;
+        matches!(self, Text | Binary | Ivm | TopLevel | Encoding | MacTab)
+    }
+
+    /// Utility function to test if the Clause is an expectation node.
+    pub fn is_expectation(&self) -> bool {
+        use ClauseType::*;
+        matches!(self, Produces | Denotes | Signals | And | Not)
+    }
+}
+
+/// Represents a valid clause accepted by the conformance DSL for specifying a test.
+#[derive(Debug)]
+pub(crate) struct Clause {
+    pub tpe: ClauseType,
+    pub body: Vec<Element>,
+}
+
+impl TryFrom<&Sequence> for Clause {
+    type Error = ConformanceErrorKind;
+
+    fn try_from(other: &Sequence) -> InnerResult<Self> {
+        let clause_type = other
+            .iter()
+            .next()
+            .ok_or(ConformanceErrorKind::UnexpectedEndOfDocument)?
+            .as_symbol()
+            .ok_or(ConformanceErrorKind::ExpectedDocumentClause)?;
+
+        let tpe = ClauseType::from_str(clause_type.text().ok_or(ConformanceErrorKind::ExpectedDocumentClause)?)?;
+        let body: Vec<Element> = other.iter().skip(1).cloned().collect();
+
+        Ok(Clause {
+            tpe,
+            body,
+        })
+    }
+}
+
+impl TryFrom<Sequence> for Clause {
+    type Error = ConformanceErrorKind;
+
+    fn try_from(other: Sequence) -> InnerResult<Self> {
+        Self::try_from(&other)
+    }
+}
+
+impl TryFrom<&[Element]> for Clause {
+    type Error = ConformanceErrorKind;
+
+    fn try_from(other: &[Element]) -> InnerResult<Self> {
+        let clause_type = other
+            .iter()
+            .next()
+            .ok_or(ConformanceErrorKind::UnexpectedEndOfDocument)?
+            .as_symbol()
+            .ok_or(ConformanceErrorKind::ExpectedDocumentClause)?;
+
+        let tpe = ClauseType::from_str(clause_type.text().ok_or(ConformanceErrorKind::ExpectedDocumentClause)?)?;
+        let body: Vec<Element> = other.iter().skip(1).cloned().collect();
+
+        Ok(Clause {
+            tpe,
+            body,
+        })
+    }
+}