More library functions

- units.parse, units.parse_bytes
- json.is_valid, json.marshal, json.unmarshal
- yaml.is_valid, yaml.marshal, yaml.unmarshal
- object.subset
- set_diff

* Also print number of errors due to each missing function
* Also lock down fully passing OPA suites

Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>

.github/workflows/rust.yml

@@ -32,3 +32,6 @@ jobs:
       run: cargo build --verbose --all-targets --target x86_64-unknown-linux-musl
     - name: Run tests (MUSL)
       run: cargo test --verbose --target x86_64-unknown-linux-musl
+    - name: Run tests (OPA Conformance)
+      run: >-
+        cargo test --test opa -- $(tr '\n' ' ' < tests/opa.passing)

Cargo.toml

@@ -32,4 +32,4 @@ debug = true
 [[test]]
 name="opa"
 harness=false
-test=false
+test=false

scripts/pre-push

@@ -14,4 +14,7 @@ if [ -f Cargo.toml ]; then
    if [[ "$OSTYPE" == "linux-gnu"* ]]; then
        scripts/coverage
    fi
+
+   # Ensure that OPA conformance tests don't regress.
+   cargo test --test opa -- $(tr '\n' ' ' < tests/opa.passing)
 fi

src/builtins/deprecated.rs

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 use crate::ast::{Expr, Ref};
-use crate::builtins::utils::ensure_args_count;
+use crate::builtins::utils::{ensure_args_count, ensure_set};
 use crate::builtins::BuiltinFcn;
 use crate::lexer::Span;
 use crate::value::Value;
@@ -19,7 +19,7 @@ lazy_static! {
 
 	m.insert("all", (all, 1));
 	m.insert("any", (any, 1));	
-
+	m.insert("set_diff", (set_diff, 2));
 	m
     };
 }
@@ -49,3 +49,11 @@ fn any(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
         }
     }))
 }
+
+fn set_diff(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "set_diff";
+    ensure_args_count(span, name, params, args, 2)?;
+    let s1 = ensure_set(name, &params[0], args[0].clone())?;
+    let s2 = ensure_set(name, &params[1], args[1].clone())?;
+    Ok(Value::from_set(s1.difference(&s2).cloned().collect()))
+}

src/builtins/encoding.rs

@@ -9,11 +9,17 @@ use crate::value::Value;
 
 use std::collections::HashMap;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use data_encoding::BASE64;
 
 pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
     m.insert("base64.decode", (base64_decode, 1));
+    m.insert("json.is_valid", (json_is_valid, 1));
+    m.insert("json.marshal", (json_marshal, 1));
+    m.insert("jsonunmarshal", (json_unmarshal, 1));
+    m.insert("yaml.is_valid", (yaml_is_valid, 1));
+    m.insert("yaml.marshal", (yaml_marshal, 1));
+    m.insert("yaml.unmarshal", (yaml_unmarshal, 1));
 }
 
 fn base64_decode(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
@@ -26,3 +32,51 @@ fn base64_decode(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Va
         String::from_utf8_lossy(&decoded_bytes).to_string(),
     ))
 }
+
+fn yaml_is_valid(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "yaml.is_valid";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let yaml_str = ensure_string(name, &params[0], &args[0])?;
+    Ok(Value::Bool(Value::from_yaml_str(&yaml_str).is_ok()))
+}
+
+fn yaml_marshal(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "yaml.marshal";
+    ensure_args_count(span, name, params, args, 1)?;
+    Ok(Value::String(
+        serde_yaml::to_string(&args[0])
+            .with_context(|| span.error("could not serialize to yaml"))?,
+    ))
+}
+
+fn yaml_unmarshal(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "yaml.unmarshal";
+    ensure_args_count(span, name, params, args, 1)?;
+    let yaml_str = ensure_string(name, &params[0], &args[0])?;
+    Value::from_yaml_str(&yaml_str).with_context(|| span.error("could not deserialize yaml."))
+}
+
+fn json_is_valid(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "json.is_valid";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let json_str = ensure_string(name, &params[0], &args[0])?;
+    Ok(Value::Bool(Value::from_json_str(&json_str).is_ok()))
+}
+
+fn json_marshal(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "json.marshal";
+    ensure_args_count(span, name, params, args, 1)?;
+    Ok(Value::String(
+        serde_json::to_string(&args[0])
+            .with_context(|| span.error("could not serialize to json"))?,
+    ))
+}
+
+fn json_unmarshal(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "json.unmarshal";
+    ensure_args_count(span, name, params, args, 1)?;
+    let json_str = ensure_string(name, &params[0], &args[0])?;
+    Value::from_json_str(&json_str).with_context(|| span.error("could not deserialize json."))
+}

src/builtins/mod.rs

@@ -17,6 +17,7 @@ mod strings;
 mod time;
 mod tracing;
 pub mod types;
+mod units;
 mod utils;
 
 use crate::ast::{Expr, Ref};
@@ -65,7 +66,7 @@ lazy_static! {
 	//opa::register(&mut m);
 	debugging::register(&mut m);
 	tracing::register(&mut m);
-
+	units::register(&mut m);
 	m
     };
 }

src/builtins/objects.rs

@@ -20,6 +20,7 @@ pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
     m.insert("object.get", (get, 3));
     m.insert("object.keys", (keys, 1));
     m.insert("object.remove", (remove, 2));
+    m.insert("object.subset", (subset, 2));
 }
 
 fn json_filter_impl(v: &Value, filter: &Value) -> Value {
@@ -202,3 +203,31 @@ fn remove(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
 
     Ok(Value::Object(obj))
 }
+
+fn is_subset(sup: &Value, sub: &Value) -> bool {
+    match (sup, sub) {
+        (Value::Object(sup), Value::Object(sub)) => {
+            sub.iter().all(|(k, vsub)| {
+                match sup.get(k) {
+                    //		    Some(vsup @ Value::Object(_)) => is_subset(vsup, vsub),
+                    Some(vsup) => is_subset(vsup, vsub),
+                    _ => false,
+                }
+            })
+        }
+        (Value::Set(sup), Value::Set(sub)) => sub.is_subset(sup),
+        (Value::Array(sup), Value::Array(sub)) => sup.windows(sub.len()).any(|w| w == &sub[..]),
+        (Value::Array(sup), Value::Set(_)) => {
+            let sup = Value::from_set(sup.iter().cloned().collect());
+            is_subset(&sup, sub)
+        }
+        (sup, sub) => sup == sub,
+    }
+}
+
+fn subset(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "object.subset";
+    ensure_args_count(span, name, params, args, 2)?;
+
+    Ok(Value::Bool(is_subset(&args[0], &args[1])))
+}

src/builtins/units.rs

@@ -0,0 +1,162 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use crate::ast::{Expr, Ref};
+use crate::builtins;
+use crate::builtins::utils::{ensure_args_count, ensure_string};
+use crate::lexer::Span;
+use crate::value::{Float, Number, Value};
+
+use std::collections::HashMap;
+
+use anyhow::{bail, Context, Result};
+use ordered_float::OrderedFloat;
+
+pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
+    m.insert("units.parse", (parse, 1));
+    m.insert("units.parse_bytes", (parse_bytes, 1));
+}
+
+fn parse(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "units.parse";
+    ensure_args_count(span, name, params, args, 1)?;
+    let string = ensure_string(name, &params[0], &args[0])?;
+    let string = string.as_str();
+
+    // Remove quotes.
+    let string = if string.starts_with('"') && string.ends_with('"') && string.len() >= 2 {
+        &string[1..string.len() - 1]
+    } else {
+        string
+    };
+
+    // Disallow whitespace.
+    if string.chars().any(char::is_whitespace) {
+        bail!(span.error("spaces not allowed in resource strings"));
+    }
+
+    let (number_part, suffix) = match string.find(|c: char| c.is_alphabetic()) {
+        Some(p) => (&string[0..p], &string[p..]),
+        _ => (string, ""),
+    };
+
+    let n: Float = if number_part.starts_with('.') {
+        serde_json::from_str(format!("0{number_part}").as_str())
+    } else {
+        serde_json::from_str(number_part)
+    }
+    .with_context(|| span.error("could not parse number"))?;
+
+    Ok(Value::Number(Number(OrderedFloat(
+        n * 10f64.powf(match suffix {
+            "E" | "e" => 18,
+            "P" | "p" => 15,
+            "T" | "t" => 12,
+            "G" | "g" => 9,
+            "M" => 6,
+            "K" | "k" => 3,
+            "m" => -3,
+
+            // The following are not supported by OPA
+            "Q" => 30,
+            "R" => 27,
+            "Y" => 24,
+            "Z" => 21,
+            "h" => 2,
+            "da" => 1,
+            "d" => -1,
+            "c" => -2,
+
+            "μ" => -6,
+            "n" => -9,
+            "f" => -15,
+            "a" => -18,
+            "z" => -21,
+            "y" => -24,
+            "r" => -27,
+            "q" => -30,
+
+            // No suffix specified.
+            "" => 0,
+            _ => {
+                return Ok(Value::Number(Number(OrderedFloat(
+                    n * 2f64.powf(match suffix.to_ascii_lowercase().as_str() {
+                        "ki" => 10,
+                        "mi" => 20,
+                        "gi" => 30,
+                        "ti" => 40,
+                        "pi" => 50,
+                        "ei" => 60,
+                        "zi" => 70,
+                        "yi" => 80,
+                        _ => return Ok(Value::Undefined),
+                    } as f64),
+                ))));
+            }
+        } as f64),
+    ))))
+}
+
+fn parse_bytes(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "units.parse_bytes";
+    ensure_args_count(span, name, params, args, 1)?;
+    let string = ensure_string(name, &params[0], &args[0])?;
+    let string = string.as_str();
+
+    // Remove quotes.
+    let string = if string.starts_with('"') && string.ends_with('"') && string.len() >= 2 {
+        &string[1..string.len() - 1]
+    } else {
+        string
+    };
+
+    // Disallow whitespace.
+    if string.chars().any(char::is_whitespace) {
+        bail!(span.error("spaces not allowed in resource strings"));
+    }
+
+    let (number_part, suffix) = match string.find(|c: char| c.is_alphabetic()) {
+        Some(p) => (&string[0..p], &string[p..]),
+        _ => (string, ""),
+    };
+
+    let n: Float = if number_part.starts_with('.') {
+        serde_json::from_str(format!("0{number_part}").as_str())
+    } else {
+        serde_json::from_str(number_part)
+    }
+    .with_context(|| span.error("could not parse number"))?;
+
+    Ok(Value::Number(Number(OrderedFloat(f64::round(
+        n * 2f64.powf(match suffix.to_ascii_lowercase().as_str() {
+            "yi" | "yib" => 80,
+            "zi" | "zib" => 70,
+            "ei" | "eib" => 60,
+            "pi" | "pib" => 50,
+            "ti" | "tib" => 40,
+            "gi" | "gib" => 30,
+            "mi" | "mib" => 20,
+            "ki" | "kib" => 10,
+            "" => 0,
+            _ => {
+                return Ok(Value::Number(Number(OrderedFloat(
+                    n * 10f64.powf(match suffix.to_ascii_lowercase().as_str() {
+                        "q" | "qb" => 30,
+                        "r" | "rb" => 27,
+                        "y" | "yb" => 24,
+                        "z" | "zb" => 21,
+                        "e" | "eb" => 18,
+                        "p" | "pb" => 15,
+                        "t" | "tb" => 12,
+                        "g" | "gb" => 9,
+                        "m" | "mb" => 6,
+                        "k" | "kb" => 3,
+                        _ => {
+                            return Ok(Value::Undefined);
+                        }
+                    } as f64),
+                ))))
+            }
+        } as f64),
+    )))))
+}