From bc821504b79f32c8bf7b226792c250e692e20b9e Mon Sep 17 00:00:00 2001 From: Julian Hyde Date: Mon, 4 Jul 2022 14:48:24 -0700 Subject: [PATCH] [MOREL-159] from should not have a singleton record type unless it ends with a singleton record yield If a `from` expression has one variable named `i` of type `int`, should the type of the returned elements be `int` or `{i: int}`? (We call `int` a scalar, `{i: int}` a singleton record type, `yield {i = i}` a singleton record yield, and `yield {i = j, j = k}` a renaming yield.) After this change, `from` will have a singleton record type only it ends with a singleton record yield. If it does not end in yield, the type depends on N, the number of pipeline variables, and will be a scalar if N = 1 and a record with N fields if N != 1. As part of this change, we introduce a new `class FromBuilder` to safely build `Core.From` pipelines. It performs micro-optimizations as it goes, such as removing `where true`, empty `order` and trivial `yield` steps. `FromBuilder` can also inline nested from expressions: from i in (from j in [1, 2, 3] where j > 1) where i < 3 becomes from j in [1, 2, 3] where j > 1 yield {i = j} where i < 3 Note the use of `yield {i = j}` to handle the variable name change caused by inlining. Ensure that `from d in scott.depts` is not printed as '' even if it is optimized to `scott.depts`. If we write `scott.depts` in the shell, the shell prints '' because `depts` may be a large table. `FromBuilder` now simplies `from d in scott.depts` to `scott.depts`, but we want the shell to print the rows, not ''. Therefore the shell now calls `Code.wrap` to tag whether an expression would be treated as a relation or as a query. Fixes #159 --- .../java/net/hydromatic/morel/ast/Ast.java | 12 +- .../java/net/hydromatic/morel/ast/Core.java | 29 +- .../net/hydromatic/morel/ast/CoreBuilder.java | 30 +- .../net/hydromatic/morel/ast/FromBuilder.java | 335 ++++++++++++++++++ .../hydromatic/morel/compile/Compiler.java | 22 +- .../hydromatic/morel/compile/Compiles.java | 22 +- .../hydromatic/morel/compile/Resolver.java | 158 ++++----- .../java/net/hydromatic/morel/eval/Codes.java | 55 ++- .../net/hydromatic/morel/foreign/Calcite.java | 10 +- .../net/hydromatic/morel/FromBuilderTest.java | 252 +++++++++++++ .../java/net/hydromatic/morel/InlineTest.java | 26 +- .../java/net/hydromatic/morel/MainTest.java | 11 +- src/test/resources/script/relational.sml | 30 ++ src/test/resources/script/relational.sml.out | 50 ++- 14 files changed, 912 insertions(+), 130 deletions(-) create mode 100644 src/main/java/net/hydromatic/morel/ast/FromBuilder.java create mode 100644 src/test/java/net/hydromatic/morel/FromBuilderTest.java diff --git a/src/main/java/net/hydromatic/morel/ast/Ast.java b/src/main/java/net/hydromatic/morel/ast/Ast.java index 186db82c..9f6eae4c 100644 --- a/src/main/java/net/hydromatic/morel/ast/Ast.java +++ b/src/main/java/net/hydromatic/morel/ast/Ast.java @@ -1502,7 +1502,6 @@ && getLast(steps) instanceof Ast.Yield) { return null; } Set fields = ImmutableSet.of(); - boolean record = true; final Set nextFields = new HashSet<>(); for (FromStep step : steps) { switch (step.op) { @@ -1517,7 +1516,6 @@ boolean record = true; } }); fields = ImmutableSet.copyOf(nextFields); - record = nextFields.size() != 1; break; case COMPUTE: @@ -1535,7 +1533,6 @@ record = nextFields.size() != 1; groupExps.forEach(pair -> nextFields.add(pair.left)); aggregates.forEach(aggregate -> nextFields.add(aggregate.id)); fields = nextFields; - record = fields.size() != 1; break; case YIELD: @@ -1546,14 +1543,15 @@ record = fields.size() != 1; .stream() .map(label -> ast.id(Pos.ZERO, label)) .collect(Collectors.toSet()); - record = true; - } else { - record = false; } break; } } - if (!record) { + + if (fields.size() == 1 + && (steps.isEmpty() + || getLast(steps).op != Op.YIELD + || ((Yield) getLast(steps)).exp.op != Op.RECORD)) { return Iterables.getOnlyElement(fields); } else { final SortedMap map = new TreeMap<>(ORDERING); diff --git a/src/main/java/net/hydromatic/morel/ast/Core.java b/src/main/java/net/hydromatic/morel/ast/Core.java index d1edf098..c19d2378 100644 --- a/src/main/java/net/hydromatic/morel/ast/Core.java +++ b/src/main/java/net/hydromatic/morel/ast/Core.java @@ -482,7 +482,7 @@ public Type type() { * value. What would be an {@code Id} in Ast is often a {@link String} in * Core; for example, compare {@link Ast.Con0Pat#tyCon} * with {@link Con0Pat#tyCon}. */ - public static class Id extends Exp { + public static class Id extends Exp implements Comparable { public final NamedPat idPat; /** Creates an Id. */ @@ -491,6 +491,10 @@ public static class Id extends Exp { this.idPat = requireNonNull(idPat); } + @Override public int compareTo(Id o) { + return idPat.compareTo(o.idPat); + } + @Override public int hashCode() { return idPat.hashCode(); } @@ -768,6 +772,17 @@ public static class Tuple extends Exp { this.args = ImmutableList.copyOf(args); } + @Override public boolean equals(Object o) { + return this == o + || o instanceof Tuple + && args.equals(((Tuple) o).args) + && type.equals(((Tuple) o).type); + } + + @Override public int hashCode() { + return Objects.hash(args, type); + } + @Override public RecordLikeType type() { return (RecordLikeType) type; } @@ -977,6 +992,16 @@ public static class From extends Exp { this.steps = requireNonNull(steps); } + @Override public boolean equals(Object o) { + return this == o + || o instanceof From + && steps.equals(((From) o).steps); + } + + @Override public int hashCode() { + return steps.hashCode(); + } + @Override public ListType type() { return (ListType) type; } @@ -1002,7 +1027,7 @@ public static class From extends Exp { public Exp copy(TypeSystem typeSystem, List steps) { return steps.equals(this.steps) ? this - : core.from(type(), steps); + : core.fromBuilder(typeSystem).addAll(steps).build(); } } diff --git a/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java b/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java index 66b8c49f..2e66a0ba 100644 --- a/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java +++ b/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java @@ -336,21 +336,26 @@ public Core.From from(ListType type, List steps) { /** Derives the result type, then calls * {@link #from(ListType, List)}. */ public Core.From from(TypeSystem typeSystem, List steps) { - final Type elementType; - if (!steps.isEmpty() && Iterables.getLast(steps) instanceof Core.Yield) { - elementType = ((Core.Yield) Iterables.getLast(steps)).exp.type; + final Type elementType = fromElementType(typeSystem, steps); + return from(typeSystem.listType(elementType), steps); + } + + /** Returns the element type of a {@link Core.From} with the given steps. */ + static Type fromElementType(TypeSystem typeSystem, + List steps) { + if (!steps.isEmpty() + && Iterables.getLast(steps) instanceof Core.Yield) { + return ((Core.Yield) Iterables.getLast(steps)).exp.type; } else { final List lastBindings = core.lastBindings(steps); if (lastBindings.size() == 1) { - elementType = getOnlyElement(lastBindings).id.type; - } else { - final SortedMap argNameTypes = new TreeMap<>(ORDERING); - lastBindings.forEach(b -> argNameTypes.put(b.id.name, b.id.type)); - elementType = typeSystem.recordType(argNameTypes); + return lastBindings.get(0).id.type; } + final SortedMap argNameTypes = new TreeMap<>(ORDERING); + lastBindings + .forEach(b -> argNameTypes.put(b.id.name, b.id.type)); + return typeSystem.recordType(argNameTypes); } - final ListType type = typeSystem.listType(elementType); - return from(type, ImmutableList.copyOf(steps)); } /** Returns what would be the yield expression if we created a @@ -385,6 +390,11 @@ public List lastBindings(List steps) { : Iterables.getLast(steps).bindings; } + /** Creates a builder that will create a {@link Core.From}. */ + public FromBuilder fromBuilder(TypeSystem typeSystem) { + return new FromBuilder(typeSystem); + } + public Core.Fn fn(FnType type, Core.IdPat idPat, Core.Exp exp) { return new Core.Fn(type, idPat, exp); } diff --git a/src/main/java/net/hydromatic/morel/ast/FromBuilder.java b/src/main/java/net/hydromatic/morel/ast/FromBuilder.java new file mode 100644 index 00000000..c91bbfc4 --- /dev/null +++ b/src/main/java/net/hydromatic/morel/ast/FromBuilder.java @@ -0,0 +1,335 @@ +/* + * Licensed to Julian Hyde under one or more contributor license + * agreements. See the NOTICE file distributed with this work + * for additional information regarding copyright ownership. + * Julian Hyde licenses this file to you under the Apache + * License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package net.hydromatic.morel.ast; + +import net.hydromatic.morel.compile.Compiles; +import net.hydromatic.morel.type.Binding; +import net.hydromatic.morel.type.TypeSystem; +import net.hydromatic.morel.util.Pair; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import org.apache.calcite.util.Util; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; + +import static net.hydromatic.morel.ast.CoreBuilder.core; + +import static com.google.common.collect.Iterables.getLast; + +/** Builds a {@link Core.From}. + * + *

Simplifies the following patterns: + *

    + *
  • Converts "from v in list" to "list" + * (only works in {@link #buildSimplify()}, not {@link #build()}); + *
  • Removes "where true" steps; + *
  • Removes empty "order" steps; + *
  • Removes trivial {@code yield}, + * e.g. "from v in list where condition yield v" + * becomes "from v in list where condition"; + *
  • Inlines {@code from} expressions, + * e.g. "from v in (from w in list)" + * becomes "from w in list yield {v = w}". + *
+ */ +public class FromBuilder { + private final TypeSystem typeSystem; + private final List steps = new ArrayList<>(); + private final List bindings = new ArrayList<>(); + + /** If non-negative, flags that particular step should be removed if it is not + * the last step. (For example, "yield {i = i}", which changes the result + * shape if the last step but is otherwise a no-op.) */ + private int removeIfNotLastIndex = Integer.MIN_VALUE; + /** If non-negative, flags that particular step should be removed if it is + * the last step. (For example, we flatten "from p in (from q in list)", + * to "from q in list yield {p = q}" but we want to remove "yield {p = q}" + * if it turns out to be the last step.) */ + private int removeIfLastIndex = Integer.MIN_VALUE; + + /** Use + * {@link net.hydromatic.morel.ast.CoreBuilder#fromBuilder(TypeSystem)}. */ + FromBuilder(TypeSystem typeSystem) { + this.typeSystem = typeSystem; + } + + /** Returns the bindings available after the most recent step. */ + public List bindings() { + return ImmutableList.copyOf(bindings); + } + + private FromBuilder addStep(Core.FromStep step) { + if (removeIfNotLastIndex == steps.size() - 1) { + // A trivial record yield with a single yield, e.g. 'yield {i = i}', has + // a purpose only if it is the last step. (It forces the return to be a + // record, e.g. '{i: int}' rather than a scalar 'int'.) + // We've just about to add a new step, so this is no longer necessary. + removeIfNotLastIndex = Integer.MIN_VALUE; + removeIfLastIndex = Integer.MIN_VALUE; + final Core.FromStep lastStep = getLast(steps); + if (lastStep.op == Op.YIELD) { + final Core.Yield yield = (Core.Yield) lastStep; + if (yield.exp.op == Op.TUPLE) { + final Core.Tuple tuple = (Core.Tuple) yield.exp; + if (tuple.args.size() == 1 && isTrivial(tuple)) { + steps.remove(steps.size() - 1); + } + } + } + } + steps.add(step); + if (!bindings.equals(step.bindings)) { + bindings.clear(); + bindings.addAll(step.bindings); + } + return this; + } + + public FromBuilder scan(Core.Pat pat, Core.Exp exp) { + return scan(pat, exp, core.boolLiteral(true)); + } + + public FromBuilder scan(Core.Pat pat, Core.Exp exp, Core.Exp condition) { + if (exp.op == Op.FROM + && steps.isEmpty() + && core.boolLiteral(true).equals(condition) + && (pat instanceof Core.IdPat + && !((Core.From) exp).steps.isEmpty() + && getLast(((Core.From) exp).steps).bindings.size() == 1 + || pat instanceof Core.RecordPat + && ((Core.RecordPat) pat).args.stream() + .allMatch(a -> a instanceof Core.IdPat))) { + final Core.From from = (Core.From) exp; + final Map nameExps = new LinkedHashMap<>(); + if (pat instanceof Core.RecordPat) { + final Core.RecordPat recordPat = (Core.RecordPat) pat; + Pair.forEach(recordPat.type().argNameTypes.keySet(), recordPat.args, + (name, arg) -> nameExps.put(name, core.id((Core.IdPat) arg))); + } else { + final Core.IdPat idPat = (Core.IdPat) pat; + final Core.FromStep lastStep = getLast(from.steps); + if (lastStep instanceof Core.Yield + && ((Core.Yield) lastStep).exp.op != Op.RECORD) { + // The last step is a yield scalar, say 'yield x + 1'. + // Translate it to a yield singleton record, say 'yield {y = x + 1}' + addAll(Util.skipLast(from.steps)); + if (((Core.Yield) lastStep).exp.op == Op.ID) { + // The last step is 'yield e'. Skip it. + return this; + } + nameExps.put(idPat.name, ((Core.Yield) lastStep).exp); + return yield_(true, core.record(typeSystem, nameExps)); + } + final Binding binding = Iterables.getOnlyElement(lastStep.bindings); + nameExps.put(idPat.name, core.id(binding.id)); + } + addAll(from.steps); + return yield_(true, core.record(typeSystem, nameExps)); + } + Compiles.acceptBinding(typeSystem, pat, bindings); + return addStep(core.scan(Op.INNER_JOIN, bindings, pat, exp, condition)); + } + + public FromBuilder addAll(Iterable steps) { + final StepHandler stepHandler = new StepHandler(); + steps.forEach(stepHandler::accept); + return this; + } + + public FromBuilder where(Core.Exp condition) { + if (condition.op == Op.BOOL_LITERAL + && (Boolean) ((Core.Literal) condition).value) { + // skip "where true" + return this; + } + return addStep(core.where(bindings, condition)); + } + + public FromBuilder group(SortedMap groupExps, + SortedMap aggregates) { + return addStep(core.group(groupExps, aggregates)); + } + + public FromBuilder order(Iterable orderItems) { + final List orderItemList = ImmutableList.copyOf(orderItems); + if (orderItemList.isEmpty()) { + // skip empty "order" + return this; + } + return addStep(core.order(bindings, orderItems)); + } + + public FromBuilder yield_(Core.Exp exp) { + return yield_(false, exp); + } + + public FromBuilder yield_(boolean uselessIfLast, Core.Exp exp) { + boolean uselessIfNotLast = false; + switch (exp.op) { + case TUPLE: + final TupleType tupleType = tupleType((Core.Tuple) exp); + switch (tupleType) { + case IDENTITY: + // A trivial record does not rename, so its only purpose is to change + // from a scalar to a record, and even then only when a singleton. + if (bindings.size() == 1) { + // Singleton record that does not rename, e.g. 'yield {x=x}' + // It only has meaning as the last step. + uselessIfNotLast = true; + break; + } else { + // Non-singleton record that does not rename, e.g. 'yield {x=x,y=y}' + // It is useless. + return this; + } + case RENAME: + if (bindings.size() == 1) { + // Singleton record that renames, e.g. 'yield {y=x}'. + // It is always useful. + break; + } else { + // Non-singleton record that renames, e.g. 'yield {y=x,z=y}' + // It is always useful. + break; + } + } + break; + + case ID: + if (bindings.size() == 1 + && ((Core.Id) exp).idPat.equals(bindings.get(0).id)) { + return this; + } + } + addStep(core.yield_(typeSystem, exp)); + removeIfNotLastIndex = uselessIfNotLast ? steps.size() - 1 : Integer.MIN_VALUE; + removeIfLastIndex = uselessIfLast ? steps.size() - 1 : Integer.MIN_VALUE; + return this; + } + + /** Returns whether tuple is something like "{i = i, j = j}". */ + private boolean isTrivial(Core.Tuple tuple) { + return tupleType(tuple) == TupleType.IDENTITY; + } + + /** Returns whether tuple is something like "{i = i, j = j}". */ + private TupleType tupleType(Core.Tuple tuple) { + if (tuple.args.size() != bindings.size()) { + return TupleType.OTHER; + } + final ImmutableList argNames = + ImmutableList.copyOf(tuple.type().argNameTypes().keySet()); + boolean identity = true; + for (int i = 0; i < tuple.args.size(); i++) { + Core.Exp exp = tuple.args.get(i); + if (exp.op != Op.ID) { + return TupleType.OTHER; + } + if (!((Core.Id) exp).idPat.name.equals(argNames.get(i))) { + identity = false; + } + } + return identity ? TupleType.IDENTITY : TupleType.RENAME; + } + + /** Returns whether tuple is something like "{i = j, j = x}". */ + private boolean isRename(Core.Tuple tuple) { + for (int i = 0; i < tuple.args.size(); i++) { + Core.Exp exp = tuple.args.get(i); + if (exp.op != Op.ID) { + return false; + } + } + return true; + } + + private Core.Exp build(boolean simplify) { + if (removeIfLastIndex == steps.size() - 1) { + removeIfLastIndex = Integer.MIN_VALUE; + final Core.Yield yield = (Core.Yield) getLast(steps); + assert yield.exp.op == Op.TUPLE + && ((Core.Tuple) yield.exp).args.size() == 1 + && isTrivial((Core.Tuple) yield.exp); + steps.remove(steps.size() - 1); + } + if (simplify + && steps.size() == 1 + && steps.get(0).op == Op.INNER_JOIN) { + final Core.Scan scan = (Core.Scan) steps.get(0); + if (scan.pat.op == Op.ID_PAT) { + return scan.exp; + } + } + return core.from(typeSystem, steps); + } + + public Core.From build() { + return (Core.From) build(false); + } + + /** As {@link #build}, but also simplifies "from x in list" to "list". */ + public Core.Exp buildSimplify() { + return build(true); + } + + /** Calls the method to re-register a step. */ + private class StepHandler extends Visitor { + @Override protected void visit(Core.Group group) { + group(group.groupExps, group.aggregates); + } + + @Override protected void visit(Core.Order order) { + order(order.orderItems); + } + + @Override protected void visit(Core.Scan scan) { + scan(scan.pat, scan.exp, scan.condition); + } + + @Override protected void visit(Core.Where where) { + where(where.exp); + } + + @Override protected void visit(Core.Yield yield) { + yield_(yield.exp); + } + } + + /** Category of expression passed to "yield". */ + private enum TupleType { + /** Tuple whose right side are the current fields, + * e.g. "{a = deptno, b = dname}". */ + RENAME, + /** Tuple whose right side are the current fields + * and left side are the same as the right, + * e.g. "{deptno = deptno, dname = dname}". */ + IDENTITY, + /** Any other tuple, + * e.g. "{a = deptno + 1, dname = dname}", + * "{deptno = deptno}" (too few fields). */ + OTHER + } +} + +// End FromBuilder.java diff --git a/src/main/java/net/hydromatic/morel/compile/Compiler.java b/src/main/java/net/hydromatic/morel/compile/Compiler.java index 53c5a269..08ee7590 100644 --- a/src/main/java/net/hydromatic/morel/compile/Compiler.java +++ b/src/main/java/net/hydromatic/morel/compile/Compiler.java @@ -45,6 +45,7 @@ import net.hydromatic.morel.util.ThreadLocals; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; import org.apache.calcite.util.Util; @@ -54,6 +55,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.function.BiConsumer; @@ -82,12 +84,12 @@ public Compiler(TypeSystem typeSystem) { } CompiledStatement compileStatement(Environment env, Core.Decl decl, - boolean isDecl) { + boolean isDecl, Set queriesToWrap) { final List matchCodes = new ArrayList<>(); final List bindings = new ArrayList<>(); final List actions = new ArrayList<>(); final Context cx = Context.of(env); - compileDecl(cx, decl, isDecl, matchCodes, bindings, actions); + compileDecl(cx, decl, isDecl, queriesToWrap, matchCodes, bindings, actions); final Type type = decl instanceof Core.NonRecValDecl ? ((Core.NonRecValDecl) decl).pat.type : PrimitiveType.UNIT; @@ -482,7 +484,8 @@ private Applicable compileApplicable(Context cx, Core.Exp fn, Type argType, private Code compileLet(Context cx, Core.Let let) { final List matchCodes = new ArrayList<>(); final List bindings = new ArrayList<>(); - compileValDecl(cx, let.decl, true, matchCodes, bindings, null); + compileValDecl(cx, let.decl, true, ImmutableSet.of(), matchCodes, bindings, + null); Context cx2 = cx.bindAll(bindings); final Code resultCode = compile(cx2, let.exp); return finishCompileLet(cx2, matchCodes, resultCode, let.type); @@ -501,12 +504,14 @@ private Code compileLocal(Context cx, Core.Local local) { } void compileDecl(Context cx, Core.Decl decl, boolean isDecl, - List matchCodes, List bindings, List actions) { + Set queriesToWrap, List matchCodes, + List bindings, List actions) { switch (decl.op) { case VAL_DECL: case REC_VAL_DECL: final Core.ValDecl valDecl = (Core.ValDecl) decl; - compileValDecl(cx, valDecl, isDecl, matchCodes, bindings, actions); + compileValDecl(cx, valDecl, isDecl, queriesToWrap, matchCodes, bindings, + actions); break; case DATATYPE_DECL: @@ -612,7 +617,8 @@ private Pair compileMatch(Context cx, Core.Match match) { } private void compileValDecl(Context cx, Core.ValDecl valDecl, boolean isDecl, - List matchCodes, List bindings, List actions) { + Set queriesToWrap, List matchCodes, + List bindings, List actions) { Compiles.bindPattern(typeSystem, bindings, valDecl); final List newBindings = new TailList<>(bindings); final Map linkCodes = new HashMap<>(); @@ -629,7 +635,9 @@ private void compileValDecl(Context cx, Core.ValDecl valDecl, boolean isDecl, // Using 'compileArg' rather than 'compile' encourages CalciteCompiler // to use a pure Calcite implementation if possible, and has no effect // in the basic Compiler. - final Code code = compileArg(cx1, exp); + final Code code0 = compileArg(cx1, exp); + final Code code = + queriesToWrap.contains(exp) ? Codes.wrapRelList(code0) : code0; if (!linkCodes.isEmpty()) { link(linkCodes, pat, code); } diff --git a/src/main/java/net/hydromatic/morel/compile/Compiles.java b/src/main/java/net/hydromatic/morel/compile/Compiles.java index 749ab54c..bd3decad 100644 --- a/src/main/java/net/hydromatic/morel/compile/Compiles.java +++ b/src/main/java/net/hydromatic/morel/compile/Compiles.java @@ -21,6 +21,7 @@ import net.hydromatic.morel.ast.Ast; import net.hydromatic.morel.ast.AstNode; import net.hydromatic.morel.ast.Core; +import net.hydromatic.morel.ast.Op; import net.hydromatic.morel.ast.Pos; import net.hydromatic.morel.ast.Visitor; import net.hydromatic.morel.eval.Prop; @@ -33,6 +34,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.ArrayList; @@ -141,7 +143,25 @@ private static CompiledStatement prepareDecl(TypeSystem typeSystem, } else { compiler = new Compiler(typeSystem); } - return compiler.compileStatement(env, coreDecl, isDecl); + + // If the user wrote "scott.depts" we will print ""; + // but if the user wrote "from d in scott.depts", they would like to see + // the full contents. Those two expressions may have been simplified to the + // same Core.Exp, but in the latter case we will 'wrap' the RelList value + // as a regular List so that it is printed in full. + final ImmutableSet.Builder queriesToWrap = ImmutableSet.builder(); + if (resolved.originalNode instanceof Ast.ValDecl + && coreDecl instanceof Core.NonRecValDecl) { + final Ast.ValDecl valDecl = (Ast.ValDecl) resolved.originalNode; + final Ast.ValBind valBind = valDecl.valBinds.get(0); + final Core.NonRecValDecl nonRecValDecl = (Core.NonRecValDecl) coreDecl; + if (valBind.exp.op == Op.FROM) { + queriesToWrap.add(nonRecValDecl.exp); + } + } + + return compiler.compileStatement(env, coreDecl, isDecl, + queriesToWrap.build()); } /** Checks for exhaustive and redundant patterns, and throws if there are diff --git a/src/main/java/net/hydromatic/morel/compile/Resolver.java b/src/main/java/net/hydromatic/morel/compile/Resolver.java index 645b2e63..f74792e5 100644 --- a/src/main/java/net/hydromatic/morel/compile/Resolver.java +++ b/src/main/java/net/hydromatic/morel/compile/Resolver.java @@ -20,6 +20,7 @@ import net.hydromatic.morel.ast.Ast; import net.hydromatic.morel.ast.Core; +import net.hydromatic.morel.ast.FromBuilder; import net.hydromatic.morel.ast.Op; import net.hydromatic.morel.ast.Pos; import net.hydromatic.morel.ast.Visitor; @@ -52,7 +53,8 @@ import java.util.function.Function; import static net.hydromatic.morel.ast.CoreBuilder.core; -import static net.hydromatic.morel.util.Static.append; + +import static com.google.common.base.Preconditions.checkArgument; /** Converts AST expressions to Core expressions. */ public class Resolver { @@ -595,87 +597,12 @@ private Core.Match toCore(Ast.Match match) { } Core.Exp toCore(Ast.From from) { - final List bindings = new ArrayList<>(); final Type type = typeMap.getType(from); - if (from.isCompute()) { - final ListType listType = typeMap.typeSystem.listType(type); - final Core.From coreFrom = - fromStepToCore(bindings, listType, from.steps, - ImmutableList.of()); - return core.only(typeMap.typeSystem, from.pos, coreFrom); - } else { - return fromStepToCore(bindings, (ListType) type, from.steps, - ImmutableList.of()); - } - } - - private Core.From fromStepToCore(List bindings, ListType type, - List steps, List coreSteps) { - final Resolver r = withEnv(bindings); - if (steps.isEmpty()) { - return core.from(type, coreSteps); - } - final Ast.FromStep step = steps.get(0); - switch (step.op) { - case SCAN: - case INNER_JOIN: - final Ast.Scan scan = (Ast.Scan) step; - final Core.Exp coreExp = r.toCore(scan.exp); - final ListType listType = (ListType) coreExp.type; - final Core.Pat corePat = r.toCore(scan.pat, listType.elementType); - final Op op = step.op == Op.SCAN - ? Op.INNER_JOIN - : step.op; - final List bindings2 = new ArrayList<>(bindings); - Compiles.acceptBinding(typeMap.typeSystem, corePat, bindings2); - Core.Exp coreCondition = scan.condition == null - ? core.boolLiteral(true) - : r.withEnv(bindings2).toCore(scan.condition); - final Core.Scan coreScan = - core.scan(op, bindings2, corePat, coreExp, coreCondition); - return fromStepToCore(coreScan.bindings, type, - Util.skip(steps), append(coreSteps, coreScan)); - - case WHERE: - final Ast.Where where = (Ast.Where) step; - final Core.Where coreWhere = - core.where(bindings, r.toCore(where.exp)); - return fromStepToCore(coreWhere.bindings, type, - Util.skip(steps), append(coreSteps, coreWhere)); - - case YIELD: - final Ast.Yield yield = (Ast.Yield) step; - final Core.Yield coreYield = - core.yield_(typeMap.typeSystem, r.toCore(yield.exp)); - return fromStepToCore(coreYield.bindings, type, - Util.skip(steps), append(coreSteps, coreYield)); - - case ORDER: - final Ast.Order order = (Ast.Order) step; - final Core.Order coreOrder = - core.order(bindings, transform(order.orderItems, r::toCore)); - return fromStepToCore(coreOrder.bindings, type, - Util.skip(steps), append(coreSteps, coreOrder)); - - case GROUP: - case COMPUTE: - final Ast.Group group = (Ast.Group) step; - final ImmutableSortedMap.Builder groupExps = - ImmutableSortedMap.naturalOrder(); - final ImmutableSortedMap.Builder aggregates = - ImmutableSortedMap.naturalOrder(); - Pair.forEach(group.groupExps, (id, exp) -> - groupExps.put(toCorePat(id), r.toCore(exp))); - group.aggregates.forEach(aggregate -> - aggregates.put(toCorePat(aggregate.id), r.toCore(aggregate))); - final Core.Group coreGroup = - core.group(groupExps.build(), aggregates.build()); - return fromStepToCore(coreGroup.bindings, type, - Util.skip(steps), append(coreSteps, coreGroup)); - - default: - throw new AssertionError("unknown step type " + step.op); - } + final Core.Exp coreFrom = new FromResolver().run(from); + checkArgument(coreFrom.type.equals(type), + "Conversion to core did not preserve type: expected [%s] " + + "actual [%s] from [%s]", type, coreFrom.type, coreFrom); + return coreFrom; } private Core.Aggregate toCore(Ast.Aggregate aggregate) { @@ -839,6 +766,75 @@ public Core.DatatypeDecl toDecl() { } } + /** Visitor that converts {@link Ast.From} to {@link Core.From} by + * handling each subtype of {@link Ast.FromStep} calling + * {@link FromBuilder} appropriately. */ + private class FromResolver extends Visitor { + final FromBuilder fromBuilder = core.fromBuilder(typeMap.typeSystem); + + Core.Exp run(Ast.From from) { + from.steps.forEach(this::accept); + final Core.Exp e = fromBuilder.buildSimplify(); + if (from.isCompute()) { + return core.only(typeMap.typeSystem, from.pos, e); + } else { + return e; + } + } + + @Override protected void visit(Ast.Scan scan) { + final Resolver r = withEnv(fromBuilder.bindings()); + final Core.Exp coreExp; + final Core.Pat corePat; + switch (scan.exp.op) { + default: + coreExp = r.toCore(scan.exp); + final ListType listType = (ListType) coreExp.type; + corePat = r.toCore(scan.pat, listType.elementType); + } + final Op op = scan.op == Op.SCAN ? Op.INNER_JOIN + : scan.op; + final List bindings2 = new ArrayList<>(fromBuilder.bindings()); + Compiles.acceptBinding(typeMap.typeSystem, corePat, bindings2); + Core.Exp coreCondition = scan.condition == null + ? core.boolLiteral(true) + : r.withEnv(bindings2).toCore(scan.condition); + fromBuilder.scan(corePat, coreExp, coreCondition); + } + + @Override protected void visit(Ast.Where where) { + final Resolver r = withEnv(fromBuilder.bindings()); + fromBuilder.where(r.toCore(where.exp)); + } + + @Override protected void visit(Ast.Yield yield) { + final Resolver r = withEnv(fromBuilder.bindings()); + fromBuilder.yield_(r.toCore(yield.exp)); + } + + @Override protected void visit(Ast.Order order) { + final Resolver r = withEnv(fromBuilder.bindings()); + fromBuilder.order(transform(order.orderItems, r::toCore)); + } + + @Override protected void visit(Ast.Compute compute) { + visit((Ast.Group) compute); + } + + @Override protected void visit(Ast.Group group) { + final Resolver r = withEnv(fromBuilder.bindings()); + final ImmutableSortedMap.Builder groupExps = + ImmutableSortedMap.naturalOrder(); + final ImmutableSortedMap.Builder aggregates = + ImmutableSortedMap.naturalOrder(); + Pair.forEach(group.groupExps, (id, exp) -> + groupExps.put(toCorePat(id), r.toCore(exp))); + group.aggregates.forEach(aggregate -> + aggregates.put(toCorePat(aggregate.id), r.toCore(aggregate))); + fromBuilder.group(groupExps.build(), aggregates.build()); + } + } + } // End Resolver.java diff --git a/src/main/java/net/hydromatic/morel/eval/Codes.java b/src/main/java/net/hydromatic/morel/eval/Codes.java index ff516ad3..d5d985e1 100644 --- a/src/main/java/net/hydromatic/morel/eval/Codes.java +++ b/src/main/java/net/hydromatic/morel/eval/Codes.java @@ -24,6 +24,7 @@ import net.hydromatic.morel.compile.BuiltIn; import net.hydromatic.morel.compile.Environment; import net.hydromatic.morel.compile.Macro; +import net.hydromatic.morel.foreign.RelList; import net.hydromatic.morel.type.Binding; import net.hydromatic.morel.type.ListType; import net.hydromatic.morel.type.PrimitiveType; @@ -47,6 +48,7 @@ import com.google.common.primitives.Chars; import org.apache.calcite.runtime.FlatLists; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -77,7 +79,8 @@ private Codes() {} /** Describes a {@link Code}. */ public static String describe(Code code) { - return code.describe(new DescriberImpl()).toString(); + final Code code2 = Codes.strip(code); + return code2.describe(new DescriberImpl()).toString(); } /** Value of {@code NONE}. @@ -311,6 +314,18 @@ public static Code orElse(Code code0, Code code1) { /** @see BuiltIn#INTERACT_USE */ private static final Applicable INTERACT_USE = new InteractUse(Pos.ZERO); + /** Removes wrappers, in particular the one due to + * {@link #wrapRelList(Code)}. */ + public static Code strip(Code code) { + for (;;) { + if (code instanceof WrapRelList) { + code = ((WrapRelList) code).code; + } else { + return code; + } + } + } + /** Implements {@link BuiltIn#INTERACT_USE}. */ private static class InteractUse extends ApplicableImpl implements Positioned { @@ -438,6 +453,10 @@ public static Code tuple(Iterable codes) { return new TupleCode(ImmutableList.copyOf(codes)); } + public static Code wrapRelList(Code code) { + return new WrapRelList(code); + } + /** Returns an applicable that constructs an instance of a datatype. * The instance is a list with two elements [constructorName, value]. */ public static Applicable tyCon(Type dataType, String name) { @@ -3549,6 +3568,40 @@ static class ApplyCodeCode implements Code { } } + /** A {@code Code} that evaluates a {@code Code} and if the result is a + * {@link net.hydromatic.morel.foreign.RelList}, wraps it in a different + * kind of list. */ + static class WrapRelList implements Code { + public final Code code; + + WrapRelList(Code code) { + this.code = code; + } + + @Override public Describer describe(Describer describer) { + return describer.start("wrapRelList", d -> d.arg("code", code)); + } + + @Override public Object eval(EvalEnv env) { + final Object arg = code.eval(env); + if (arg instanceof RelList) { + final RelList list = (RelList) arg; + return new AbstractList() { + @Override public Object get(int index) { + return list.get(index); + } + + @Override public int size() { + return list.size(); + } + }; + } + return arg; + } + } + + + /** An {@link Applicable} whose position can be changed. * *

Operations that may throw exceptions should implement this interface. diff --git a/src/main/java/net/hydromatic/morel/foreign/Calcite.java b/src/main/java/net/hydromatic/morel/foreign/Calcite.java index 519a51eb..df810dc3 100644 --- a/src/main/java/net/hydromatic/morel/foreign/Calcite.java +++ b/src/main/java/net/hydromatic/morel/foreign/Calcite.java @@ -99,7 +99,9 @@ public Code code(Environment env, RelNode rel, Type type) { final RelNode rel2 = program.run(planner, rel, traitSet, ImmutableList.of(), ImmutableList.of()); - return new CalciteCode(dataContext, rel2, type, env); + final Function, List> converter = + Converters.fromEnumerable(rel, type); + return new CalciteCode(dataContext, rel2, env, converter); } /** Copied from {@link Programs}. */ @@ -173,12 +175,12 @@ private static class CalciteCode implements Code { final Environment env; final Function, List> converter; - CalciteCode(DataContext dataContext, RelNode rel, Type type, - Environment env) { + CalciteCode(DataContext dataContext, RelNode rel, Environment env, + Function, List> converter) { this.dataContext = dataContext; this.rel = rel; this.env = env; - converter = Converters.fromEnumerable(rel, type); + this.converter = converter; } // to help with debugging diff --git a/src/test/java/net/hydromatic/morel/FromBuilderTest.java b/src/test/java/net/hydromatic/morel/FromBuilderTest.java new file mode 100644 index 00000000..f38691ae --- /dev/null +++ b/src/test/java/net/hydromatic/morel/FromBuilderTest.java @@ -0,0 +1,252 @@ +/* + * Licensed to Julian Hyde under one or more contributor license + * agreements. See the NOTICE file distributed with this work + * for additional information regarding copyright ownership. + * Julian Hyde licenses this file to you under the Apache + * License, Version 2.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package net.hydromatic.morel; + +import net.hydromatic.morel.ast.Ast; +import net.hydromatic.morel.ast.Core; +import net.hydromatic.morel.ast.FromBuilder; +import net.hydromatic.morel.type.PrimitiveType; +import net.hydromatic.morel.type.TypeSystem; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; + +import static net.hydromatic.morel.ast.CoreBuilder.core; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Test {@link net.hydromatic.morel.ast.FromBuilder}. + */ +public class FromBuilderTest { + private static class Fixture { + final TypeSystem typeSystem = new TypeSystem(); + final PrimitiveType intType = PrimitiveType.INT; + final Core.IdPat aPat = core.idPat(intType, "a", 0); + final Core.Id aId = core.id(aPat); + final Core.IdPat bPat = core.idPat(intType, "b", 0); + final Core.IdPat iPat = core.idPat(intType, "i", 0); + final Core.Id iId = core.id(iPat); + final Core.IdPat jPat = core.idPat(intType, "j", 0); + final Core.Id jId = core.id(jPat); + final Core.Exp list12 = core.list(typeSystem, intType, + ImmutableList.of(intLiteral(1), intLiteral(2))); + final Core.Exp list34 = core.list(typeSystem, intType, + ImmutableList.of(intLiteral(3), intLiteral(4))); + + Core.Literal intLiteral(int i) { + return core.literal(intType, i); + } + + Core.Exp record(Core.Id... ids) { + final Map nameExps = new LinkedHashMap<>(); + Arrays.asList(ids).forEach(id -> nameExps.put(id.idPat.name, id)); + return core.record(typeSystem, nameExps); + } + } + + @Test void testBasic() { + // from i in [1, 2] + final Fixture f = new Fixture(); + final FromBuilder fromBuilder = core.fromBuilder(f.typeSystem); + fromBuilder.scan(f.iPat, f.list12); + + final Core.From from = fromBuilder.build(); + assertThat(from.toString(), is("from i in [1, 2]")); + final Core.Exp e = fromBuilder.buildSimplify(); + assertThat(e.toString(), is("[1, 2]")); + + // "from i in [1, 2] yield i" --> "[1, 2]" + fromBuilder.yield_(f.iId); + final Core.From from2 = fromBuilder.build(); + assertThat(from2, is(from)); + final Core.Exp e2 = fromBuilder.buildSimplify(); + assertThat(e2, is(e)); + } + + @Test void testWhereOrder() { + // from i in [1, 2] where i < 2 order i desc + // ==> + // from i in [1, 2] + final Fixture f = new Fixture(); + final FromBuilder fromBuilder = core.fromBuilder(f.typeSystem); + fromBuilder.scan(f.iPat, f.list12) + .where(core.lessThan(f.typeSystem, f.iId, f.intLiteral(2))) + .order(ImmutableList.of(core.orderItem(f.iId, Ast.Direction.DESC))); + + final Core.From from = fromBuilder.build(); + assertThat(from.toString(), + is("from i in [1, 2] where i < 2 order i desc")); + final Core.Exp e = fromBuilder.buildSimplify(); + assertThat(e, is(from)); + + // "where true" and "order {}" are ignored + fromBuilder.where(core.boolLiteral(true)) + .order(ImmutableList.of()) + .where(core.greaterThan(f.typeSystem, f.iId, f.intLiteral(1))); + final Core.From from2 = fromBuilder.build(); + assertThat(from2.toString(), + is("from i in [1, 2] where i < 2 order i desc where i > 1")); + final Core.Exp e2 = fromBuilder.buildSimplify(); + assertThat(e2, is(from2)); + } + + @Test void testTrivialYield() { + // from i in [1, 2] where i < 2 yield i + final Fixture f = new Fixture(); + final FromBuilder fromBuilder = core.fromBuilder(f.typeSystem); + fromBuilder.scan(f.iPat, f.list12) + .where(core.lessThan(f.typeSystem, f.iId, f.intLiteral(2))) + .yield_(f.iId); + + final Core.From from = fromBuilder.build(); + assertThat(from.toString(), is("from i in [1, 2] where i < 2")); + final Core.Exp e = fromBuilder.buildSimplify(); + assertThat(e, is(from)); + } + + @Test void testTrivialYield2() { + // from j in [1, 2], i in [3, 4] where i < 2 yield {i, j} + // ==> + // from j in [1, 2], i in [3, 4] where i < 2 + final Fixture f = new Fixture(); + final FromBuilder fromBuilder = core.fromBuilder(f.typeSystem); + fromBuilder.scan(f.jPat, f.list12) + .scan(f.iPat, f.list34) + .where(core.lessThan(f.typeSystem, f.iId, f.intLiteral(2))) + .yield_(f.record(f.iId, f.jId)); + + final Core.From from = fromBuilder.build(); + final String expected = "from j in [1, 2] " + + "join i in [3, 4] " + + "where i < 2"; + assertThat(from.toString(), is(expected)); + final Core.Exp e = fromBuilder.buildSimplify(); + assertThat(e, is(from)); + } + + @Test void testNested() { + // from i in (from j in [1, 2] where j < 2) where i > 1 + // ==> + // from j in [1, 2] where j < 2 yield {i = j} where i > 1 + final Fixture f = new Fixture(); + final Core.From innerFrom = + core.fromBuilder(f.typeSystem) + .scan(f.jPat, f.list12) + .where(core.lessThan(f.typeSystem, f.jId, f.intLiteral(2))) + .build(); + + final FromBuilder fromBuilder = core.fromBuilder(f.typeSystem); + fromBuilder.scan(f.iPat, innerFrom) + .where(core.greaterThan(f.typeSystem, f.iId, f.intLiteral(1))); + + final Core.From from = fromBuilder.build(); + final String expected = "from j in [1, 2] " + + "where j < 2 " + + "yield {i = j} " + + "where i > 1"; + assertThat(from.toString(), is(expected)); + final Core.Exp e = fromBuilder.buildSimplify(); + assertThat(e, is(from)); + } + + /** As {@link #testNested()} but inner and outer variables have the same + * name, and therefore no yield is required. */ + @Test void testNestedSameName() { + // from i in (from i in [1, 2] where i < 2) where i > 1 + // ==> + // from i in [1, 2] where i < 2 where i > 1 + final Fixture f = new Fixture(); + final Core.From innerFrom = + core.fromBuilder(f.typeSystem) + .scan(f.iPat, f.list12) + .where(core.lessThan(f.typeSystem, f.iId, f.intLiteral(2))) + .build(); + + final FromBuilder fromBuilder = core.fromBuilder(f.typeSystem); + fromBuilder.scan(f.iPat, innerFrom) + .where(core.greaterThan(f.typeSystem, f.iId, f.intLiteral(1))); + + final Core.From from = fromBuilder.build(); + final String expected = "from i in [1, 2] " + + "where i < 2 " + + "where i > 1"; + assertThat(from.toString(), is(expected)); + final Core.Exp e = fromBuilder.buildSimplify(); + assertThat(e, is(from)); + } + + @Test void testNested0() { + // from i in (from) + // ==> + // from + final Fixture f = new Fixture(); + final Core.From innerFrom = + core.fromBuilder(f.typeSystem) + .build(); + + final FromBuilder fromBuilder = core.fromBuilder(f.typeSystem); + fromBuilder.scan(f.iPat, innerFrom); + + final Core.From from = fromBuilder.build(); + assertThat(from.toString(), is("from i in (from)")); + final Core.Exp e = fromBuilder.buildSimplify(); + assertThat(e.toString(), is("from")); + } + + @Test void testNested2() { + // from {i = a, j = b} in (from a in [1, 2], b in [3, 4] where a < 2) + // where i < j + // ==> + // from a in [1, 2], b in [3, 4] where a < 2 yield {i = a, j = b} + // where i < j + final Fixture f = new Fixture(); + final Core.From innerFrom = + core.fromBuilder(f.typeSystem) + .scan(f.aPat, f.list12) + .scan(f.bPat, f.list34) + .where(core.lessThan(f.typeSystem, f.aId, f.intLiteral(2))) + .build(); + + final FromBuilder fromBuilder = core.fromBuilder(f.typeSystem); + fromBuilder.scan( + core.recordPat(f.typeSystem, ImmutableSet.of("i", "j"), + ImmutableList.of(f.aPat, f.bPat)), + innerFrom) + .where(core.lessThan(f.typeSystem, f.iId, f.jId)); + + final Core.From from = fromBuilder.build(); + final String expected = "from a in [1, 2] " + + "join b in [3, 4] " + + "where a < 2 " + + "yield {i = a, j = b} " + + "where i < j"; + assertThat(from.toString(), is(expected)); + final Core.Exp e = fromBuilder.buildSimplify(); + assertThat(e, is(from)); + } +} + +// End FromBuilderTest.java diff --git a/src/test/java/net/hydromatic/morel/InlineTest.java b/src/test/java/net/hydromatic/morel/InlineTest.java index f212916b..c7982a9a 100644 --- a/src/test/java/net/hydromatic/morel/InlineTest.java +++ b/src/test/java/net/hydromatic/morel/InlineTest.java @@ -222,7 +222,6 @@ private String v(int i) { final String core2 = "val it = " + "from e in #emp scott " + "where op mod (#empno e, 2) = 0 " - + "yield {e = e} " + "where #deptno e_1 = 10 " + "yield #ename e_1"; ml(ml) @@ -245,9 +244,9 @@ private String v(int i) { + "#filter List (fn e => #deptno e = 30) (#emp scott) " + "yield (fn e_1 => #empno e_1) v0"; final String core2 = "val it = " - + "from v2 in #emp scott " - + "where #deptno v2 = 30 " - + "yield {v0 = v2} " + + "from v3 in #emp scott " + + "where #deptno v3 = 30 " + + "yield {v0 = v3} " + "yield #empno v0"; ml(ml) .withBinding("scott", BuiltInDataSet.SCOTT) @@ -278,13 +277,13 @@ private String v(int i) { + " (#filter List (fn e => #deptno e = 30) (#emp scott)))) " + "yield (fn r_2 => r_2 + 100) v0"; final String core2 = "val it = " - + "from v6 in #emp scott " - + "where #deptno v6 = 30 " - + "yield {v5 = v6} " - + "yield {v4 = {x = #empno v5, y = #deptno v5, z = 15}} " - + "where #y v4 > #z v4 " - + "yield {v2 = v4} " - + "yield {v0 = #x v2 + #z v2} " + + "from v13 in #emp scott " + + "where #deptno v13 = 30 " + + "yield {v10 = v13} " + + "yield {v7 = {x = #empno v10, y = #deptno v10, z = 15}} " + + "where #y v7 > #z v7 " + + "yield {v3 = v7} " + + "yield {v0 = #x v3 + #z v3} " + "yield v0 + 100"; ml(ml) .withBinding("scott", BuiltInDataSet.SCOTT) @@ -299,9 +298,8 @@ private String v(int i) { + "where i > 10\n" + "yield i / 10"; final String core0 = "val it = " - + "from i in" - + " (from e in #emp scott" - + " yield #deptno e) " + + "from e in #emp scott " + + "yield {i = #deptno e} " + "where i > 10 " + "yield i / 10"; final String core1 = "val it = " diff --git a/src/test/java/net/hydromatic/morel/MainTest.java b/src/test/java/net/hydromatic/morel/MainTest.java index 88c70276..4fea16ce 100644 --- a/src/test/java/net/hydromatic/morel/MainTest.java +++ b/src/test/java/net/hydromatic/morel/MainTest.java @@ -1767,10 +1767,19 @@ private static List node(Object... args) { final String ml = "from e in [{x=1,y=2},{x=3,y=4},{x=5,y=6}]\n" + " yield {z=e.x}\n" + " where z > 2\n" - + " order z desc"; + + " order z desc\n" + + " yield {z=z}"; ml(ml) .assertType("{z:int} list") .assertEvalIter(equalsOrdered(list(5), list(3))); + + final String ml2 = "from e in [{x=1,y=2},{x=3,y=4},{x=5,y=6}]\n" + + " yield {z=e.x}\n" + + " where z > 2\n" + + " order z desc"; + ml(ml2) + .assertType("int list") + .assertEvalIter(equalsOrdered(5, 3)); } /** Analogous to SQL "CROSS APPLY" which calls a table-valued function diff --git a/src/test/resources/script/relational.sml b/src/test/resources/script/relational.sml index e316b70d..fcbe1dad 100644 --- a/src/test/resources/script/relational.sml +++ b/src/test/resources/script/relational.sml @@ -64,6 +64,10 @@ from e in emps yield {deptno = e.deptno, one = 1}; from e in emps yield {e.deptno, one = 1}; +from e in emps yield {x = e.deptno} where x > 10 yield {y = x} where y < 30; + +from e in emps yield {x = e.deptno} where x > 10 yield {x = x} where x < 30; + from e in emps yield ((#id e) + (#deptno e)); from e in emps yield (e.id + e.deptno); @@ -102,6 +106,11 @@ from e in emps yield {e.deptno, e.name} order deptno desc; +from e in emps + yield {e.deptno} + order deptno desc + yield {deptno}; + from e in emps yield {e.deptno} order deptno desc; @@ -123,6 +132,12 @@ from e in emps yield {d = e.deptno} where d > 10; +(*) singleton record 'yield' followed by 'where' followed by 'yield' +from e in emps + yield {d = e.deptno} + where d > 10 + yield {d = d}; + (*) singleton record 'yield' followed by singleton 'group' from e in emps yield {d = e.deptno} @@ -140,6 +155,12 @@ from e in emps yield {d = e.deptno} order d desc; +(*) singleton record 'yield' followed by 'order' then singleton 'yield' +from e in emps + yield {d = e.deptno} + order d desc + yield {d = d}; + (*) singleton record 'yield' followed by 'order' then 'yield' from e in emps yield {d = e.deptno} @@ -733,6 +754,15 @@ from where true; from where false; +from i in (from j in (from)); + +from i in (from), + j in (from); + +from i in (from), + j in (from) + yield j; + let val b = 1 < 2 in diff --git a/src/test/resources/script/relational.sml.out b/src/test/resources/script/relational.sml.out index 760ca4e8..6d53d9b9 100644 --- a/src/test/resources/script/relational.sml.out +++ b/src/test/resources/script/relational.sml.out @@ -126,6 +126,14 @@ val it = : {deptno:int, one:int} list +from e in emps yield {x = e.deptno} where x > 10 yield {y = x} where y < 30; +val it = [20] : int list + + +from e in emps yield {x = e.deptno} where x > 10 yield {x = x} where x < 30; +val it = [20] : int list + + from e in emps yield ((#id e) + (#deptno e)); val it = [110,121,132,133] : int list @@ -193,10 +201,17 @@ val it = from e in emps yield {e.deptno} - order deptno desc; + order deptno desc + yield {deptno}; val it = [{deptno=30},{deptno=30},{deptno=20},{deptno=10}] : {deptno:int} list +from e in emps + yield {e.deptno} + order deptno desc; +val it = [30,30,20,10] : int list + + (*) 'yield' followed by 'order' from e in emps yield {e.deptno, x = e.deptno, e.name} @@ -223,6 +238,14 @@ val it = from e in emps yield {d = e.deptno} where d > 10; +val it = [20,30,30] : int list + + +(*) singleton record 'yield' followed by 'where' followed by 'yield' +from e in emps + yield {d = e.deptno} + where d > 10 + yield {d = d}; val it = [{d=20},{d=30},{d=30}] : {d:int} list @@ -246,6 +269,14 @@ val it = [{c=1,d=10},{c=1,d=20},{c=2,d=30}] : {c:int, d:int} list from e in emps yield {d = e.deptno} order d desc; +val it = [30,30,20,10] : int list + + +(*) singleton record 'yield' followed by 'order' then singleton 'yield' +from e in emps + yield {d = e.deptno} + order d desc + yield {d = d}; val it = [{d=30},{d=30},{d=20},{d=10}] : {d:int} list @@ -1163,6 +1194,21 @@ from where false; val it = [] : unit list +from i in (from j in (from)); +val it = [()] : unit list + + +from i in (from), + j in (from); +val it = [{i=(),j=()}] : {i:unit, j:unit} list + + +from i in (from), + j in (from) + yield j; +val it = [()] : unit list + + let val b = 1 < 2 in @@ -1670,7 +1716,7 @@ val it = ["Shaggy","Scooby"] : string list Sys.plan(); val it = - "from(sink join(op join, pat e_2, exp from(sink join(op join, pat e, exp constant([[10, 100, Fred], [20, 101, Velma], [30, 102, Shaggy], [30, 103, Scooby]]), sink where(condition apply2(fnValue =, apply(fnValue nth:0, argCode get(name e)), constant(30)), sink collect(get(name e))))), sink collect(apply(fnValue nth:2, argCode get(name e)))))" + "from(sink join(op join, pat e, exp constant([[10, 100, Fred], [20, 101, Velma], [30, 102, Shaggy], [30, 103, Scooby]]), sink where(condition apply2(fnValue =, apply(fnValue nth:0, argCode get(name e)), constant(30)), sink collect(apply(fnValue nth:2, argCode get(name e))))))" : string