Merges DML and other changes made for QLDB

This includes:

- Support for DML statements INSERT, UPDATE, DELETE, etc).
- Support for the BY FROM-source alias
- A preliminary static type checker
- A number of minor bug fixes and refactorings.

Does *not* include DDL statements (CREATE, DROP, and UNDROP), as well as
refactorings and bug fixes.

Contributors to these changes include:  Almann Goo, Rob Calhoun,
David Lurton, Michael Ross and James Bowman.

Commit subject-lines below:

- Add AstNodeTest tests for DML nodes.
- Fix issues remaining after merge
- Merge branch 'quantum-dev' of IonSQLSandbox into quantum-dev
- Merge branch 'mainline' into quantum-dev
- improve ambiguous binding error message (for real this time)
- Revert "Improve ambiguous binding error message"
- Improve ambiguous binding error message
- introduce SelectStarRewriter
- fix LIMIT with value over 2^31 returning no results
- StaticTypeRewriter now requires FromSourceAliasRewriter
- fix variable lookup within FROM sources
- optimize IN operator
- IDs in DML Simple Paths
- Allow UPDATE form to be used for any DML Operation
- add unit tests for CROSS JOIN with condition and other join types without conditions
- Restricts various JOINs as per PartiQL spec.
- Add support for BY to legacy DML
- Add enhanced validations to StaticTypeRewriter
- Lowercase the id of the count VariableReference
- Capture Token for parse types that recurr on tail
- Support FROM source BY variables in ExprValue and EvaluatingCompliler
- From source BY variable support in parser and (de)serializer.
- Introduces FromSourceLet and LetVariables to the AST
- Add support for V0 to `AstSerializer` and make `V0AstSerializer` a shim
- Add support for DML to StaticTypeRewriter
- Delete zero byte BindingCase.* files which somehow were added to the project root
- Merge in rename of com.amazon.ionsql to org.partiql from mainline
- Add generic type arguments to Bindings references that aren't in the lang project
- Bring quantum-dev in line with 'mainline'
- Reorganize directory structure to match mainline.
- Merge in minor changes to package structure.
- Eliminates MapBindings<T> and fixes StaticTypeRewriter
- Merges in changes to StaticTypeRewriter.

cli/README.md

@@ -1,4 +1,4 @@
-# PartiQL CLI
+ # PartiQL CLI
 
 Command line interface for executing PartiQL queries.
 

cli/src/org/partiql/cli/Cli.kt

@@ -29,14 +29,14 @@ internal class Cli(private val valueFactory: ExprValueFactory,
                    private val output: OutputStream,
                    private val format: OutputFormat,
                    private val compilerPipeline: CompilerPipeline,
-                   private val globals: Bindings,
+                   private val globals: Bindings<ExprValue>,
                    private val query: String) : PartiQLCommand {
 
     override fun run() {
         IonReaderBuilder.standard().build(input).use { reader ->
             val inputIonValue = valueFactory.ion.iterate(reader).asSequence().map { valueFactory.newFromIonValue(it) }
             val inputExprValue = valueFactory.newBag(inputIonValue)
-            val bindings = Bindings.buildLazyBindings {
+            val bindings = Bindings.buildLazyBindings<ExprValue> {
                 addBinding("input_data") { inputExprValue }
             }.delegate(globals)
 

cli/src/org/partiql/cli/Repl.kt

@@ -28,7 +28,7 @@ internal const val PROMPT_1 = "PartiQL> "
 internal const val PROMPT_2 = "   | "
 internal const val BAR_1 = "===' "
 internal const val BAR_2 = "--- "
-internal const val WELCOME_MSG = "Welcome to the PartiQL REPL!" // TODO: extract version from gradle.build and append to message 
+internal const val WELCOME_MSG = "Welcome to the PartiQL REPL!" // TODO: extract version from gradle.build and append to message
 
 private enum class ReplState {
     /** Initial state, first state as soon as you start the REPL */
@@ -61,18 +61,20 @@ private enum class ReplState {
 
 private class GlobalBinding(private val valueFactory: ExprValueFactory) {
 
+
+
     private val knownNames = mutableSetOf<String>()
-    var bindings = Bindings.EMPTY
+    var bindings = Bindings.empty<ExprValue>()
         private set
 
-    fun add(bindings: Bindings): GlobalBinding {
+    fun add(bindings: Bindings<ExprValue>): GlobalBinding {
         when (bindings) {
             is MapBindings -> {
-                knownNames.addAll(bindings.bindingMap.originalCaseMap.keys)
+                knownNames.addAll(bindings.originalCaseMap.keys)
                 this.bindings = bindings.delegate(this.bindings)
             }
-            Bindings.EMPTY -> {
-            } // nothing to do 
+            Bindings.empty<ExprValue>() -> {
+            } // nothing to do
             else           -> throw IllegalArgumentException("Invalid binding type for global environment: $bindings")
         }
 
@@ -106,8 +108,9 @@ internal class Repl(private val valueFactory: ExprValueFactory,
                     output: OutputStream,
                     private val parser: Parser,
                     private val compiler: CompilerPipeline,
-                    initialGlobal: Bindings,
-                    private val timer: Timer = object : Timer {}) : PartiQLCommand {
+                    initialGlobal: Bindings<ExprValue>,
+                    private val timer: Timer = object : Timer {}
+) : PartiQLCommand {
 
     private val outputWriter = OutputStreamWriter(output, "UTF-8")
 
@@ -124,16 +127,16 @@ internal class Repl(private val valueFactory: ExprValueFactory,
                 throw IllegalArgumentException("add_to_global_env requires 1 parameter")
             }
 
-            val locals = Bindings.buildLazyBindings { addBinding("_") { previousResult } }.delegate(globals.bindings)
+            val locals = Bindings.buildLazyBindings<ExprValue> { addBinding("_") { previousResult } }.delegate(globals.bindings)
             val result = compiler.compile(source).eval(EvaluationSession.build { globals(locals) })
             globals.add(result.bindings)
 
             return result
         }
 
-        private fun globalEnv(source: String): ExprValue? = globals.asExprValue()
+        private fun globalEnv(@Suppress("UNUSED_PARAMETER") source: String): ExprValue? = globals.asExprValue()
 
-        private fun listCommands(source: String): ExprValue? {
+        private fun listCommands(@Suppress("UNUSED_PARAMETER") source: String): ExprValue? {
             outputWriter.write("\n")
             outputWriter.write("""
                 |!add_to_global_env: adds a value to the global environment
@@ -168,7 +171,7 @@ internal class Repl(private val valueFactory: ExprValueFactory,
     private val valuePrettyPrinter = ConfigurableExprValueFormatter.pretty
     private val astPrettyPrinter = object : ExprValueFormatter {
         val writer = IonTextWriterBuilder.pretty().build(outputWriter)
-        
+
         override fun formatTo(value: ExprValue, out: Appendable) {
             value.ionValue.writeTo(writer)
             writer.flush()
@@ -210,7 +213,7 @@ internal class Repl(private val valueFactory: ExprValueFactory,
                 if (result != null) {
                     outputWriter.write(BAR_1)
                     outputWriter.write("\n")
-                    
+
                     formatter.formatTo(result, outputWriter)
                     outputWriter.write("\n")
 
@@ -240,7 +243,7 @@ internal class Repl(private val valueFactory: ExprValueFactory,
 
     private fun executePartiQL(): ReplState = executeTemplate(valuePrettyPrinter) { source ->
         if (source != "") {
-            val locals = Bindings.buildLazyBindings { addBinding("_") { previousResult } }.delegate(globals.bindings)
+            val locals = Bindings.buildLazyBindings<ExprValue> { addBinding("_") { previousResult } }.delegate(globals.bindings)
 
             compiler.compile(source).eval(EvaluationSession.build { globals(locals) })
         }
@@ -251,7 +254,7 @@ internal class Repl(private val valueFactory: ExprValueFactory,
 
     private fun parsePartiQL(): ReplState = executeTemplate(astPrettyPrinter) { source ->
         if (source != "") {
-            val serializedAst = AstSerializer.serialize(parser.parseExprNode(source), valueFactory.ion)
+            val serializedAst = AstSerializer.serialize(parser.parseExprNode(source), AstVersion.V1, valueFactory.ion)
             valueFactory.newFromIonValue(serializedAst)
         }
         else {
@@ -261,7 +264,7 @@ internal class Repl(private val valueFactory: ExprValueFactory,
 
     private fun parsePartiQLWithFilters(): ReplState = executeTemplate(astPrettyPrinter) { source ->
         if (source != "") {
-            val serializedAst = AstSerializer.serialize(parser.parseExprNode(source), valueFactory.ion)
+            val serializedAst = AstSerializer.serialize(parser.parseExprNode(source), AstVersion.V1, valueFactory.ion)
             valueFactory.newFromIonValue(serializedAst.filterTermNodes())
         }
         else {

cli/src/org/partiql/cli/main.kt

@@ -130,7 +130,7 @@ fun main(args: Array<String>) = try {
             val config = compilerPipeline.compile(configSource).eval(EvaluationSession.standard())
             config.bindings
         }
-        else                          -> Bindings.EMPTY
+        else                          -> Bindings.empty()
     }
 
     if (optionSet.has(queryOpt)) {
@@ -151,11 +151,11 @@ catch (e: Exception) {
     exitProcess(1)
 }
 
-private fun runRepl(environment: Bindings) {
+private fun runRepl(environment: Bindings<ExprValue>) {
     Repl(valueFactory, System.`in`, System.out, parser, compilerPipeline, environment).run()
 }
 
-private fun runCli(environment: Bindings, optionSet: OptionSet) {
+private fun runCli(environment: Bindings<ExprValue>, optionSet: OptionSet) {
     val input = if (optionSet.has(inputFileOpt)) {
         FileInputStream(optionSet.valueOf(inputFileOpt))
     }

cli/test/org/partiql/cli/CliTest.kt

@@ -32,7 +32,7 @@ class CliTest {
         output.reset()
     }
 
-    private fun makeCli(query: String, input: String, bindings: Bindings = Bindings.EMPTY) =
+    private fun makeCli(query: String, input: String, bindings: Bindings<ExprValue> = Bindings.empty()) =
         Cli(
             valueFactory,
             input.byteInputStream(Charsets.UTF_8),

cli/test/org/partiql/cli/ReplTest.kt

@@ -61,7 +61,7 @@ class RequiredFlushOutputStream : OutputStream() {
     override fun toString() = backingOS.toString("UTF-8")
 }
 
-private class ReplTester(bindings: Bindings = Bindings.EMPTY) {
+private class ReplTester(bindings: Bindings<ExprValue> = Bindings.empty()) {
     val ion = IonSystemBuilder.standard().build()
     val parser: Parser = SqlParser(ion)
     val compiler = CompilerPipeline.build(ion) { sqlParser(parser) }
@@ -112,7 +112,7 @@ private class ReplTester(bindings: Bindings = Bindings.EMPTY) {
             Thread.sleep(SLEEP_TIME)
         }
 
-        // wait for the REPL to print the initial message and prompt 
+        // wait for the REPL to print the initial message and prompt
         outputPhaser.arriveAndAwaitAdvance()
 
         val inputLines = extractInputLines(expectedPromptText)
@@ -123,20 +123,20 @@ private class ReplTester(bindings: Bindings = Bindings.EMPTY) {
             // flush to repl input
             inputPipe.flush()
 
-            // wait for output to be written before inputting more 
+            // wait for output to be written before inputting more
             outputPhaser.arriveAndAwaitAdvance()
         }
 
         // nothing more to write
         inputPipe.close()
 
-        // make sure output was written 
+        // make sure output was written
         outputPhaser.arriveAndAwaitAdvance()
 
         assertEquals(expectedPromptText, actualReplPrompt.toString())
     }
 
-    private fun extractInputLines(expectedPromptText: String): List<String> = 
+    private fun extractInputLines(expectedPromptText: String): List<String> =
         expectedPromptText.split("\n")
             .filter { line ->
                 line.startsWith(PROMPT_2) || (line.startsWith(PROMPT_1) && line != PROMPT_1)
@@ -351,7 +351,7 @@ class ReplTest {
             .compile("{'foo': [1,2,3]}")
             .eval(EvaluationSession.standard())
             .bindings
-        
+
         ReplTester(initialBindings).assertReplPrompt("""
             #Welcome to the PartiQL REPL!
             #PartiQL> !global_env

cli/test/org/partiql/cli/functions/ReadFileTest.kt

@@ -24,7 +24,7 @@ class ReadFileTest {
     private val ion = IonSystemBuilder.standard().build()
     private val valueFactory = ExprValueFactory.standard(ion)
     private val function = ReadFile(valueFactory)
-    private val env = Environment(locals = Bindings.EMPTY,
+    private val env = Environment(locals = Bindings.empty(),
                                   session = EvaluationSession.standard())
 
     private fun String.exprValue() = valueFactory.newFromIonValue(ion.singleValue(this))

cli/test/org/partiql/cli/functions/WriteFileTest.kt

@@ -24,7 +24,7 @@ class WriteFileTest {
     private val ion = IonSystemBuilder.standard().build()
     private val valueFactory = ExprValueFactory.standard(ion)
     private val function = WriteFile(valueFactory)
-    private val env = Environment(locals = Bindings.EMPTY,
+    private val env = Environment(locals = Bindings.empty(),
                                   session = EvaluationSession.standard())
 
     private fun String.exprValue() = valueFactory.newFromIonValue(ion.singleValue(this))

docs/dev/README-AST-INTRO.md

@@ -14,7 +14,7 @@ Language implementations often use the term "Abstract Syntax Tree" to refer to a
 graph. Our implementation for PartiQL's AST is a tree and *not* a graph. It contains no cycles and each node can only
 reference its children.
 
-##  It's Immutable
+## It's Immutable
 
 No mechanism has been provided to mutate an instance of the AST after it has been instantiated.  Modifications to an
 existing tree must use cloning re-writes--for instance, a visitor pattern that returns a modified version of the input
@@ -48,9 +48,12 @@ The base classes are:
 - `org.partiql.lang.ast.SelectListItem`, for expressions that may appear between `SELECT` ... `FROM`.
 - `org.partiql.lang.ast.FromSource`, for expressions that are data sources in a `FROM` clause.
 - `org.partiql.lang.ast.PathComponent`, for the components of a path expression.
-- `org.partiql.lang.ast.SelectProjection`, - indicates the type of projection used by a
+- `org.partiql.lang.ast.SelectProjection`, for the type of projection used by a
 `SELECT`,`SELECT VALUE` or `PIVOT` query. This isn't directly related to the grammar but is convenient to represent
 in this manner.
+- `org.partiql.lang.ast.DataManipulation`, for data manipulation expressions that may optionally be wrapped with
+ `FROM ... WHERE ...`.
+- `org.partiql.lang.DmlOperation`, for the data manipulation operation itself (e.g. `INSERT INTO ...`)
 
 All base classes of the AST are [sealed](https://kotlinlang.org/docs/reference/sealed-classes.html) classes.
 

examples/src/java/org/partiql/examples/CSVJavaExample.java

@@ -115,7 +115,7 @@ public void run() {
 
         final EvaluationSession session = EvaluationSession.builder()
                 .globals(
-                        Bindings.lazyBindingsBuilder().addBinding("myCsvDocument", () -> {
+                        Bindings.<ExprValue>lazyBindingsBuilder().addBinding("myCsvDocument", () -> {
                             List<CsvRowExprValue> csvValues = Arrays.stream(CSV.split("\n"))
                                     .map(csvLine -> new CsvRowExprValue(pipeline.getValueFactory(), csvLine))
                                     .collect(Collectors.toList());

examples/src/java/org/partiql/examples/S3Example.java

@@ -79,7 +79,7 @@ public void run() {
                     // We implement the Bindings interface using a lambda. Bindings are used to map names into values,
                     // in this case we are binding the data from the S3 bucket into the "myS3Document" name
                     .globals(
-                            Bindings.lazyBindingsBuilder()
+                            Bindings.<ExprValue>lazyBindingsBuilder()
                                     .addBinding("myS3Document", () -> pipeline.getValueFactory().newFromIonValue(values))
                                     .build()
                     )

examples/src/kotlin/org/partiql/examples/CsvExprValueExample.kt

@@ -57,7 +57,7 @@ private class CsvRowExprValue(private val valueFactory: ExprValueFactory, privat
         Bindings.ofMap(rowValues)
     }
 
-    override val bindings: Bindings
+    override val bindings: Bindings<ExprValue>
         get() = bindingsInstance
 }
 
@@ -78,7 +78,7 @@ class CsvExprValueExample(out: PrintStream) : Example(out) {
     override fun run() {
         print("CSV file:", EXAMPLE_CSV_FILE_CONTENTS)
 
-        val globals = Bindings.buildLazyBindings {
+        val globals = Bindings.buildLazyBindings<ExprValue> {
             addBinding("csv_data") {
                 // The first time "csv_data" is encountered during query evaluation this closure will be
                 // invoked to obtain its value, which will then be cached for later use.

examples/src/kotlin/org/partiql/examples/EvaluationWithLazyBindings.kt

@@ -19,7 +19,7 @@ class EvaluationWithLazyBindings(out: PrintStream) : Example(out) {
 
         // The global bindings
         val ionText = """[ { name: "Nibbler", age: 2 }, { name: "Hobbes", age: 6 } ]"""
-        val globalVariables = Bindings.buildLazyBindings {
+        val globalVariables = Bindings.buildLazyBindings<ExprValue> {
             addBinding("pets") {
                 // The first time "pets" is encountered during query evaluation this closure will be
                 // invoked to obtain its value, which will then be cached for later use.

examples/src/kotlin/org/partiql/examples/ParserExample.kt

@@ -38,7 +38,7 @@ class ParserExample(out: PrintStream) : Example(out) {
         // Convert the ExprNode AST to the Ion s-expression form.
         // AstSerializer always serializes to the latest version of the s-expression format.
         // the serialized format is documented at `docs/dev/README-AST-V1.md`
-        val serializedAst = AstSerializer.serialize(originalAst, ion)
+        val serializedAst = AstSerializer.serialize(originalAst, AstVersion.V1, ion)
         print("Serialized AST", serializedAst.toPrettyString())
 
         // Re-constitute the serialized AST.  The deserializer will convert from any supported

examples/src/kotlin/org/partiql/examples/PartialEvaluationRewriter.kt

@@ -82,10 +82,10 @@ class PartialEvaluationRewriterExample(out: PrintStream) : Example(out) {
             addPreprocessingStep { exprNode, stepContext ->
                 val rewriter = PartialEvaluationRewriter(ion, stepContext.compileOptions)
 
-                print("Original AST:", AstSerializer.serialize(exprNode, ion).toString())
+                print("Original AST:", AstSerializer.serialize(exprNode, AstVersion.V1, ion).toString())
 
                 val rewrittenAst = rewriter.rewriteExprNode(exprNode)
-                print("Rewritten AST:", AstSerializer.serialize(rewrittenAst, ion).toString())
+                print("Rewritten AST:", AstSerializer.serialize(rewrittenAst, AstVersion.V1, ion).toString())
 
                 rewrittenAst
             }