Add Plan Evaluator

docs/user/BuiltInFunctions.md

@@ -226,8 +226,6 @@ CAST(<<'a', 'b'>> AS bag) -- <<'a', 'b'>> (REPL does not display << >> and comma
 
 ### CHAR_LENGTH, CHARACTER_LENGTH
 
-
-
 Counts the number of characters in the specified string, where 'character' is defined as a single unicode code point.
 
 *Note:* `CHAR_LENGTH` and `CHARACTER_LENGTH` are synonyms. 
@@ -455,9 +453,31 @@ EXTRACT(TIMEZONE_MINUTE FROM TIME WITH TIME ZONE '23:12:59-08:30')  -- -30
 ```
 *Note* that `timezone_hour` and `timezone_minute` are **not supported** for `DATE` and `TIME` (without time zone) type. 
 
+### `FILTER_DISTINCT`
+
+Signature
+: `FILTER_DISTINCT: Container -> Bag`
+
+Header
+: `FILTER_DISTINCT(c)`
+
+Purpose
+: Returns a bag of distinct values contained within a bag, list, sexp, or struct.  If the container is a struct,
+the field names are not considered.
+
+Examples
+:
+
+```sql
+FILTER_DISTINCT([0, 0, 1])                  -- <<0, 1>>
+FILTER_DISTINCT(<<0, 0, 1>>)                -- <<0, 1>>
+FILTER_DISTINCT(SEXP(0, 0, 1))              -- <<0, 1>>
+FILTER_DISTINCT({'a': 0, 'b': 0, 'c': 1})   -- <<0, 1>>
+```
+
 ### LOWER 
 
-Given a string convert all upper case characters to lower case characters. 
+Given a string convert all upper case characters to lower case characters.
 
 Signature
 : `LOWER: String -> String`

lang/resources/org/partiql/type-domains/partiql.ion

@@ -725,7 +725,7 @@ may then be further optimized by selecting better implementations of each operat
                     join_type::join_type
                     left::bexpr
                     right::bexpr
-                    predicate::expr)
+                    predicate::(? expr))
 
 
                 (offset i::impl row_count::expr source::bexpr)

lang/src/org/partiql/lang/CompilerPipeline.kt

@@ -39,7 +39,7 @@ import org.partiql.lang.types.StaticType
 import org.partiql.lang.util.interruptibleFold
 
 /**
- * Contains all of the information needed for processing steps.
+ * Contains all information needed for processing steps.
  */
 data class StepContext(
     /** The instance of [ExprValueFactory] that is used by the pipeline. */
@@ -102,6 +102,11 @@ interface CompilerPipeline {
      */
     val procedures: @JvmSuppressWildcards Map<String, StoredProcedure>
 
+    /**
+     * The configured global type bindings.
+     */
+    val globalTypeBindings: Bindings<StaticType>?
+
     /** Compiles the specified PartiQL query using the configured parser. */
     fun compile(query: String): Expression
 
@@ -244,7 +249,7 @@ internal class CompilerPipelineImpl(
     override val customDataTypes: List<CustomType>,
     override val procedures: Map<String, StoredProcedure>,
     private val preProcessingSteps: List<ProcessingStep>,
-    private val globalTypeBindings: Bindings<StaticType>?
+    override val globalTypeBindings: Bindings<StaticType>?
 ) : CompilerPipeline {
 
     private val compiler = EvaluatingCompiler(

lang/src/org/partiql/lang/SqlException.kt

@@ -34,12 +34,10 @@ import org.partiql.lang.util.propertyValueMapOf
  *
  * @param message the message for this exception
  * @param errorCode the error code for this exception
- * @param errorContextArg context for this error, contains details like line & column number among other attributes.
+ * @param errorContext context for this error, includes details like line & character offsets, among others.
+ * @param internal True to indicate that this exception a bug in ParitQL itself.  False to indicate it was caused by
+ * incorrect usage of APIs or invalid end-user queries.
  * @param cause for this exception
- *
- * @constructor a custom error [message], the [errorCode], error context as a [propertyValueMap] and optional [cause] creates an
- * [SqlException]. This is the constructor for the second configuration explained above.
- *
  */
 open class SqlException(
     override var message: String,
@@ -83,7 +81,7 @@ open class SqlException(
      *
      *  * ErrorCategory is one of `Lexer Error`, `Parser Error`, `Runtime Error`
      *  * ErrorLocation is the line and column where the error occurred
-     *  * Errormessatge is the **generated** error message
+     *  * ErrorMessage is the **generated** error message
      *
      *
      * TODO: Prepend to the auto-generated message the file name.
@@ -92,6 +90,10 @@ open class SqlException(
     fun generateMessage(): String =
         "${errorCategory(errorCode)}: ${errorLocation(errorContext)}: ${errorMessage(errorCode, errorContext)}"
 
+    /** Same as [generateMessage] but without the location. */
+    fun generateMessageNoLocation(): String =
+        "${errorCategory(errorCode)}: ${errorMessage(errorCode, errorContext)}"
+
     private fun errorMessage(errorCode: ErrorCode?, propertyValueMap: PropertyValueMap?): String =
         errorCode?.getErrorMessage(propertyValueMap) ?: UNKNOWN
 

lang/src/org/partiql/lang/ast/passes/SemanticException.kt

@@ -37,7 +37,7 @@ class SemanticException(
     constructor(err: Problem, cause: Throwable? = null) :
         this(
             message = "",
-            errorCode = ErrorCode.SEMANTIC_INFERENCER_ERROR,
+            errorCode = ErrorCode.SEMANTIC_PROBLEM,
             errorContext = propertyValueMapOf(
                 Property.LINE_NUMBER to err.sourceLocation.lineNum,
                 Property.COLUMN_NUMBER to err.sourceLocation.charOffset,

lang/src/org/partiql/lang/domains/util.kt

@@ -1,5 +1,6 @@
 package org.partiql.lang.domains
 
+import com.amazon.ionelement.api.IonElement
 import com.amazon.ionelement.api.MetaContainer
 import com.amazon.ionelement.api.emptyMetaContainer
 import com.amazon.ionelement.api.metaContainerOf
@@ -14,6 +15,19 @@ import org.partiql.lang.eval.BindingCase
 fun PartiqlAst.Builder.id(name: String) =
     id(name, caseInsensitive(), unqualified())
 
+// TODO:  once https://github.com/partiql/partiql-ir-generator/issues/6 has been completed, we can delete this.
+fun PartiqlLogical.Builder.id(name: String) =
+    id(name, caseInsensitive(), unqualified())
+
+// TODO:  once https://github.com/partiql/partiql-ir-generator/issues/6 has been completed, we can delete this.
+fun PartiqlLogical.Builder.pathExpr(exp: PartiqlLogical.Expr) =
+    pathExpr(exp, caseInsensitive())
+
+// Workaround for a bug in PIG that is fixed in its next release:
+// https://github.com/partiql/partiql-ir-generator/issues/41
+fun List<IonElement>.asAnyElement() =
+    this.map { it.asAnyElement() }
+
 val MetaContainer.staticType: StaticTypeMeta? get() = this[StaticTypeMeta.TAG] as StaticTypeMeta?
 
 /** Constructs a container with the specified metas. */
@@ -60,17 +74,17 @@ fun PartiqlAst.CaseSensitivity.toBindingCase(): BindingCase = when (this) {
 }
 
 /**
- * Returns the [SourceLocationMeta] as an error context if the [SourceLocationMeta.TAG] exists in the passed
- * [metaContainer]. Otherwise, returns an empty map.
+ * Converts a [PartiqlLogical.CaseSensitivity] to a [BindingCase].
  */
-fun errorContextFrom(metaContainer: MetaContainer?): PropertyValueMap {
-    if (metaContainer == null) {
-        return PropertyValueMap()
-    }
-    val location = metaContainer[SourceLocationMeta.TAG] as? SourceLocationMeta
-    return if (location != null) {
-        org.partiql.lang.eval.errorContextFrom(location)
-    } else {
-        PropertyValueMap()
-    }
+fun PartiqlLogical.CaseSensitivity.toBindingCase(): BindingCase = when (this) {
+    is PartiqlLogical.CaseSensitivity.CaseInsensitive -> BindingCase.INSENSITIVE
+    is PartiqlLogical.CaseSensitivity.CaseSensitive -> BindingCase.SENSITIVE
+}
+
+/**
+ * Converts a [PartiqlLogical.CaseSensitivity] to a [BindingCase].
+ */
+fun PartiqlPhysical.CaseSensitivity.toBindingCase(): BindingCase = when (this) {
+    is PartiqlPhysical.CaseSensitivity.CaseInsensitive -> BindingCase.INSENSITIVE
+    is PartiqlPhysical.CaseSensitivity.CaseSensitive -> BindingCase.SENSITIVE
 }

lang/src/org/partiql/lang/errors/ErrorCode.kt

@@ -647,7 +647,7 @@ enum class ErrorCode(
                 "got: ${errorContext?.get(Property.ACTUAL_ARGUMENT_TYPES) ?: UNKNOWN}"
     },
 
-    SEMANTIC_INFERENCER_ERROR(
+    SEMANTIC_PROBLEM(
         ErrorCategory.SEMANTIC,
         LOCATION + setOf(Property.MESSAGE),
         ""
@@ -980,12 +980,6 @@ enum class ErrorCode(
         ErrorBehaviorInPermissiveMode.RETURN_MISSING
     ),
 
-    EVALUATOR_SQL_EXCEPTION(
-        ErrorCategory.EVALUATOR,
-        LOCATION,
-        "SQL exception"
-    ),
-
     EVALUATOR_COUNT_START_NOT_ALLOWED(
         ErrorCategory.EVALUATOR,
         LOCATION,

lang/src/org/partiql/lang/eval/EvaluatingCompiler.kt

@@ -3058,7 +3058,7 @@ private class SingleProjectionElement(val name: ExprValue, val thunk: ThunkEnv)
  */
 private class MultipleProjectionElement(val thunks: List<ThunkEnv>) : ProjectionElement()
 
-private val MetaContainer.sourceLocationMeta get() = this[SourceLocationMeta.TAG] as? SourceLocationMeta
+internal val MetaContainer.sourceLocationMeta get() = this[SourceLocationMeta.TAG] as? SourceLocationMeta
 
 private fun StaticType.getTypes() = when (val flattened = this.flatten()) {
     is AnyOfType -> flattened.types

lang/src/org/partiql/lang/eval/builtins/BuiltinFunctions.kt

@@ -15,15 +15,20 @@
 package org.partiql.lang.eval.builtins
 
 import com.amazon.ion.system.IonSystemBuilder
+import org.partiql.lang.eval.DEFAULT_COMPARATOR
 import org.partiql.lang.eval.EvaluationSession
 import org.partiql.lang.eval.ExprFunction
 import org.partiql.lang.eval.ExprValue
 import org.partiql.lang.eval.ExprValueFactory
 import org.partiql.lang.eval.stringValue
+import org.partiql.lang.eval.unnamedValue
 import org.partiql.lang.types.AnyOfType
 import org.partiql.lang.types.FunctionSignature
 import org.partiql.lang.types.StaticType
 import org.partiql.lang.types.UnknownArguments
+import java.util.TreeSet
+
+internal const val DYNAMIC_LOOKUP_FUNCTION_NAME = "\$__dynamic_lookup__"
 
 internal fun createBuiltinFunctionSignatures(): Map<String, FunctionSignature> =
     // Creating a new IonSystem in this instance is not the problem it would normally be since we are
@@ -40,6 +45,7 @@ internal fun createBuiltinFunctions(valueFactory: ExprValueFactory) =
         createCharacterLength("character_length", valueFactory),
         createCharacterLength("char_length", valueFactory),
         createUtcNow(valueFactory),
+        createFilterDistinct(valueFactory),
         DateAddExprFunction(valueFactory),
         DateDiffExprFunction(valueFactory),
         ExtractExprFunction(valueFactory),
@@ -77,6 +83,29 @@ internal fun createUtcNow(valueFactory: ExprValueFactory): ExprFunction = object
         valueFactory.newTimestamp(session.now)
 }
 
+internal fun createFilterDistinct(valueFactory: ExprValueFactory): ExprFunction = object : ExprFunction {
+    override val signature = FunctionSignature(
+        "filter_distinct",
+        listOf(StaticType.unionOf(StaticType.BAG, StaticType.LIST, StaticType.SEXP, StaticType.STRUCT)),
+        returnType = StaticType.BAG
+    )
+
+    override fun callWithRequired(session: EvaluationSession, required: List<ExprValue>): ExprValue {
+        val argument = required.first()
+        // We cannot use a [HashSet] here because [ExprValue] does not implement .equals() and .hashCode()
+        val encountered = TreeSet(DEFAULT_COMPARATOR)
+        return valueFactory.newBag(
+            sequence {
+                argument.asSequence().forEach {
+                    if (!encountered.contains(it)) {
+                        encountered.add(it.unnamedValue())
+                        yield(it)
+                    }
+                }
+            }
+        )
+    }
+}
 internal fun createCharacterLength(name: String, valueFactory: ExprValueFactory): ExprFunction =
     object : ExprFunction {
         override val signature: FunctionSignature

lang/src/org/partiql/lang/eval/builtins/DynamicLookupExprFunction.kt

@@ -0,0 +1,101 @@
+package org.partiql.lang.eval.builtins
+
+import org.partiql.lang.errors.ErrorCode
+import org.partiql.lang.eval.BindingCase
+import org.partiql.lang.eval.BindingName
+import org.partiql.lang.eval.EvaluationException
+import org.partiql.lang.eval.EvaluationSession
+import org.partiql.lang.eval.ExprFunction
+import org.partiql.lang.eval.ExprValue
+import org.partiql.lang.eval.ExprValueType
+import org.partiql.lang.eval.physical.throwUndefinedVariableException
+import org.partiql.lang.eval.stringValue
+import org.partiql.lang.types.FunctionSignature
+import org.partiql.lang.types.StaticType
+import org.partiql.lang.types.VarargFormalParameter
+
+/**
+ * Performs dynamic variable resolution.  Query authors should never call this function directly (and indeed it is
+ * named to avoid collision with the names of custom functions)--instead, the query planner injects call sites
+ * to this function to perform dynamic variable resolution of undefined variables.  This provides a migration path
+ * for legacy customers that depend on this behavior.
+ *
+ * Arguments:
+ *
+ * - variable name
+ * - case sensitivity
+ * - lookup strategy (globals then locals or locals then globals)
+ * - A variadic list of locations to be searched.
+ *
+ * The variadic arguments must be of type `any` because the planner doesn't yet have knowledge of static types
+ * and therefore cannot filter out local variables types that are not structs.
+ */
+class DynamicLookupExprFunction : ExprFunction {
+    override val signature: FunctionSignature
+        get() {
+            return FunctionSignature(
+                name = DYNAMIC_LOOKUP_FUNCTION_NAME,
+                // Required parameters are: variable name, case sensitivity and lookup strategy
+                requiredParameters = listOf(StaticType.SYMBOL, StaticType.SYMBOL, StaticType.SYMBOL),
+                variadicParameter = VarargFormalParameter(StaticType.ANY, 0..Int.MAX_VALUE),
+                returnType = StaticType.ANY
+            )
+        }
+
+    override fun callWithVariadic(
+        session: EvaluationSession,
+        required: List<ExprValue>,
+        variadic: List<ExprValue>
+    ): ExprValue {
+        val variableName = required[0].stringValue()
+
+        val caseSensitivity = when (val caseSensitivityParameterValue = required[1].stringValue()) {
+            "case_sensitive" -> BindingCase.SENSITIVE
+            "case_insensitive" -> BindingCase.INSENSITIVE
+            else -> throw EvaluationException(
+                message = "Invalid case sensitivity: $caseSensitivityParameterValue",
+                errorCode = ErrorCode.INTERNAL_ERROR,
+                internal = true
+            )
+        }
+
+        val bindingName = BindingName(variableName, caseSensitivity)
+
+        val globalsFirst = when (val lookupStrategyParameterValue = required[2].stringValue()) {
+            "locals_then_globals" -> false
+            "globals_then_locals" -> true
+            else -> throw EvaluationException(
+                message = "Invalid lookup strategy: $lookupStrategyParameterValue",
+                errorCode = ErrorCode.INTERNAL_ERROR,
+                internal = true
+            )
+        }
+
+        val found = when {
+            globalsFirst -> {
+                session.globals[bindingName] ?: searchLocals(variadic, bindingName)
+            }
+            else -> {
+                searchLocals(variadic, bindingName) ?: session.globals[bindingName]
+            }
+        }
+
+        if (found == null) {
+            // We don't know the metas inside ExprFunction implementations.  The ThunkFactory error handlers
+            // should add line & col info to the exception & rethrow anyway.
+            throwUndefinedVariableException(bindingName, metas = null)
+        } else {
+            return found
+        }
+    }
+
+    private fun searchLocals(possibleLocations: List<ExprValue>, bindingName: BindingName) =
+        possibleLocations.asSequence().map {
+            when (it.type) {
+                ExprValueType.STRUCT ->
+                    it.bindings[bindingName]
+                else ->
+                    null
+            }
+        }.firstOrNull { it != null }
+}