Start [MOREL-75] Outer join

Add left, right, full keywords; derive type of join expressions add method Ast.Id.toPat(); move joinRelType; full join execution works Fixes hydromatic#75
julianhyde · Feb 1, 2024 · 12868ca · 12868ca
1 parent 9c3425d
commit 12868ca
Show file tree

Hide file tree

Showing 24 changed files with 609 additions and 116 deletions.
diff --git a/docs/reference.md b/docs/reference.md
@@ -151,7 +151,8 @@ In Standard ML but not in Morel:
     | <i>pat</i> <b>=</b> <i>exp</i> [ <b>on</b> <i>exp</i> ]      single iteration
     | <i>var</i>                       unbounded variable
 <i>step</i> &rarr; <b>where</b> <i>exp</i>                filter clause
-    | <b>join</b> <i>scan<sub>1</sub></i> [ <b>,</b> ... <b>,</b> <i>scan<sub>s</sub></i> ]
+    | [ <b>left</b> | <b>right</b> | <b>full</b> ]
+      <b>join</b> <i>scan<sub>1</sub></i> [ <b>,</b> ... <b>,</b> <i>scan<sub>s</sub></i> ]
                                 join clause
     | <b>group</b> <i>groupKey<sub>1</sub></i> <b>,</b> ... <b>,</b> <i>groupKey<sub>g</sub></i>
       [ <b>compute</b> <i>agg<sub>1</sub></i> <b>,</b> ... <b>,</b> <i>agg<sub>a</sub></i> ]

diff --git a/src/main/java/net/hydromatic/morel/ast/Ast.java b/src/main/java/net/hydromatic/morel/ast/Ast.java
@@ -38,13 +38,13 @@
 import java.util.SortedMap;
 import java.util.function.Consumer;
 import java.util.function.ObjIntConsumer;
-import java.util.stream.Collectors;
 
 import static net.hydromatic.morel.ast.AstBuilder.ast;
 import static net.hydromatic.morel.type.RecordType.ORDERING;
 import static net.hydromatic.morel.type.RecordType.mutableMap;
 import static net.hydromatic.morel.util.Ord.forEachIndexed;
 import static net.hydromatic.morel.util.Pair.forEachIndexed;
+import static net.hydromatic.morel.util.Static.toImmutableSet;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.Iterables.getLast;
@@ -774,6 +774,11 @@ public Id accept(Shuttle shuttle) {
     AstWriter unparse(AstWriter w, int left, int right) {
       return w.id(name);
     }
+
+    /** Creates a pattern equivalent to this identifier. */
+    public IdPat toPat() {
+      return new IdPat(pos, name);
+    }
   }
 
   /** Parse tree node of a record selector. */
@@ -949,7 +954,7 @@ public DatatypeBind accept(Shuttle shuttle) {
    */
   public static class TyCon extends AstNode {
     public final Id id;
-    public final @org.checkerframework.checker.nullness.qual.Nullable Type type;
+    public final @Nullable Type type;
 
     TyCon(Pos pos, Id id, Type type) {
       super(pos, Op.TY_CON);
@@ -1513,6 +1518,9 @@ public static class From extends Exp {
       for (FromStep step : steps) {
         switch (step.op) {
         case SCAN:
+        case LEFT_JOIN:
+        case RIGHT_JOIN:
+        case FULL_JOIN:
           final Scan scan = (Scan) step;
           nextFields.clear();
           nextFields.addAll(fields);
@@ -1549,7 +1557,7 @@ public static class From extends Exp {
           nextFields.addAll(Pair.left(groupExps));
           groupExps.forEach((id, exp) -> nextFields.add(id));
           aggregates.forEach(aggregate -> nextFields.add(aggregate.id));
-          fields = nextFields;
+          fields = ImmutableSet.copyOf(nextFields);
           break;
 
         case YIELD:
@@ -1559,7 +1567,7 @@ public static class From extends Exp {
                 ((Record) yield.exp).args.keySet()
                     .stream()
                     .map(label -> ast.id(Pos.ZERO, label))
-                    .collect(Collectors.toSet());
+                    .collect(toImmutableSet());
           }
           break;
         }
@@ -1644,8 +1652,11 @@ public static class Scan extends FromStep {
     public final @Nullable Exp exp;
     public final @Nullable Exp condition;
 
-    Scan(Pos pos, Pat pat, @Nullable Exp exp, @Nullable Exp condition) {
-      super(pos, Op.SCAN);
+    Scan(Pos pos, Op op, Pat pat, @Nullable Exp exp, @Nullable Exp condition) {
+      super(pos, op);
+      checkArgument(op == Op.SCAN || op == Op.LEFT_JOIN
+              || op == Op.RIGHT_JOIN || op == Op.FULL_JOIN,
+          "not a join type: %s", op);
       this.pat = pat;
       this.exp = exp;
       this.condition = condition;
@@ -1683,7 +1694,7 @@ public Scan copy(Pat pat, @Nullable Exp exp, @Nullable Exp condition) {
           && Objects.equals(this.exp, exp)
           && Objects.equals(this.condition, condition)
           ? this
-          : new Scan(pos, pat, exp, condition);
+          : new Scan(pos, op, pat, exp, condition);
     }
   }
 

diff --git a/src/main/java/net/hydromatic/morel/ast/AstBuilder.java b/src/main/java/net/hydromatic/morel/ast/AstBuilder.java
@@ -457,9 +457,9 @@ public Ast.Exp map(Pos pos, Ast.Exp e1, Ast.Exp e2) {
     return apply(apply(ref(pos, BuiltIn.LIST_MAP), e1), e2);
   }
 
-  public Ast.Scan scan(Pos pos, Ast.Pat pat, Ast.Exp exp,
+  public Ast.Scan scan(Pos pos, Op op, Ast.Pat pat, Ast.Exp exp,
       Ast.@Nullable Exp condition) {
-    return new Ast.Scan(pos, pat, exp, condition);
+    return new Ast.Scan(pos, op, pat, exp, condition);
   }
 
   public Ast.Order order(Pos pos, Iterable<Ast.OrderItem> orderItems) {

diff --git a/src/main/java/net/hydromatic/morel/ast/AstNodes.java b/src/main/java/net/hydromatic/morel/ast/AstNodes.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Julian Hyde under one or more contributor license
+ * agreements.  See the NOTICE file distributed with this work
+ * for additional information regarding copyright ownership.
+ * Julian Hyde licenses this file to you under the Apache
+ * License, Version 2.0 (the "License"); you may not use this
+ * file except in compliance with the License.  You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied.  See the License for the specific
+ * language governing permissions and limitations under the
+ * License.
+ */
+package net.hydromatic.morel.ast;
+
+import org.apache.calcite.rel.core.JoinRelType;
+
+/** Utilities for abstract syntax trees. */
+public abstract class AstNodes {
+  /** Converts an {@link Op} into a Calcite join type, otherwise throws. */
+  public static JoinRelType joinRelType(Op op) {
+    switch (op) {
+    case SCAN:
+      return JoinRelType.INNER;
+    case LEFT_JOIN:
+      return JoinRelType.LEFT;
+    case RIGHT_JOIN:
+      return JoinRelType.RIGHT;
+    case FULL_JOIN:
+      return JoinRelType.FULL;
+    default:
+      throw new AssertionError(op);
+    }
+  }
+}
+
+// End AstNodes.java
diff --git a/src/main/java/net/hydromatic/morel/ast/Core.java b/src/main/java/net/hydromatic/morel/ast/Core.java
@@ -1188,9 +1188,12 @@ public static class Scan extends FromStep {
     public final Exp exp;
     public final Exp condition;
 
-    Scan(ImmutableList<Binding> bindings, Pat pat, Exp exp,
+    Scan(Op op, ImmutableList<Binding> bindings, Pat pat, Exp exp,
         Exp condition) {
-      super(Op.SCAN, bindings);
+      super(op, bindings);
+      checkArgument(op == Op.SCAN || op == Op.LEFT_JOIN
+              || op == Op.RIGHT_JOIN || op == Op.FULL_JOIN,
+          "not a join type: %s", op);
       this.pat = requireNonNull(pat, "pat");
       this.exp = requireNonNull(exp, "exp");
       this.condition = requireNonNull(condition, "condition");
@@ -1233,7 +1236,7 @@ private static boolean canAssign(Type fromType, Type toType) {
             .append(exp, Op.EQ.right, 0);
       }
       if (!isLiteralTrue()) {
-        w.append("on").append(condition, 0, 0);
+        w.append(" on ").append(condition, 0, 0);
       }
       return w;
     }
@@ -1249,7 +1252,7 @@ public Scan copy(List<Binding> bindings, Pat pat, Exp exp, Exp condition) {
           && condition == this.condition
           && bindings.equals(this.bindings)
           ? this
-          : core.scan(bindings, pat, exp, condition);
+          : core.scan(op, bindings, pat, exp, condition);
     }
   }
 

diff --git a/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java b/src/main/java/net/hydromatic/morel/ast/CoreBuilder.java
@@ -491,9 +491,10 @@ public Core.DatatypeDecl datatypeDecl(Iterable<DataType> dataTypes) {
     return new Core.DatatypeDecl(ImmutableList.copyOf(dataTypes));
   }
 
-  public Core.Scan scan(List<Binding> bindings, Core.Pat pat,
+  public Core.Scan scan(Op op, List<Binding> bindings, Core.Pat pat,
       Core.Exp exp, Core.Exp condition) {
-    return new Core.Scan(ImmutableList.copyOf(bindings), pat, exp, condition);
+    return new Core.Scan(op, ImmutableList.copyOf(bindings), pat, exp,
+        condition);
   }
 
   public Core.Aggregate aggregate(Type type, Core.Exp aggregate,

diff --git a/src/main/java/net/hydromatic/morel/ast/FromBuilder.java b/src/main/java/net/hydromatic/morel/ast/FromBuilder.java
@@ -41,6 +41,7 @@
 import static net.hydromatic.morel.util.Pair.forEach;
 import static net.hydromatic.morel.util.Static.append;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.Iterables.getLast;
 
 /** Builds a {@link Core.From}.
@@ -137,16 +138,20 @@ && isTrivial(tuple, previousBindings, yield.bindings)) {
   public FromBuilder scan(Core.Pat pat) {
     final Core.Exp extent =
         core.extent(typeSystem, pat.type, ImmutableRangeSet.of(Range.all()));
-    return scan(pat, extent, core.boolLiteral(true));
+    return scan(Op.SCAN, pat, extent, core.boolLiteral(true));
   }
 
   /** Creates a bounded scan, "from pat in exp". */
   public FromBuilder scan(Core.Pat pat, Core.Exp exp) {
-    return scan(pat, exp, core.boolLiteral(true));
+    return scan(Op.SCAN, pat, exp, core.boolLiteral(true));
   }
 
-  public FromBuilder scan(Core.Pat pat, Core.Exp exp, Core.Exp condition) {
+  public FromBuilder scan(Op op, Core.Pat pat, Core.Exp exp,
+      Core.Exp condition) {
+    checkArgument(op == Op.SCAN || op == Op.LEFT_JOIN
+        || op == Op.RIGHT_JOIN || op == Op.FULL_JOIN);
     if (exp.op == Op.FROM
+        && op == Op.SCAN
         && core.boolLiteral(true).equals(condition)
         && (pat instanceof Core.IdPat
             && !((Core.From) exp).steps.isEmpty()
@@ -208,7 +213,7 @@ && getLast(((Core.From) exp).steps).bindings.size() == 1
       return yield_(uselessIfLast, bindings, core.record(typeSystem, nameExps));
     }
     Compiles.acceptBinding(typeSystem, pat, bindings);
-    return addStep(core.scan(bindings, pat, exp, condition));
+    return addStep(core.scan(op, bindings, pat, exp, condition));
   }
 
   public FromBuilder addAll(Iterable<? extends Core.FromStep> steps) {
@@ -403,7 +408,7 @@ private class StepHandler extends Visitor {
     }
 
     @Override protected void visit(Core.Scan scan) {
-      scan(scan.pat, scan.exp, scan.condition);
+      scan(scan.op, scan.pat, scan.exp, scan.condition);
     }
 
     @Override protected void visit(Core.Where where) {

diff --git a/src/main/java/net/hydromatic/morel/ast/Op.java b/src/main/java/net/hydromatic/morel/ast/Op.java
@@ -20,7 +20,7 @@
 
 import com.google.common.collect.ImmutableMap;
 
-/** Sub-types of {@link AstNode}. */
+/** Subtypes of {@link AstNode}. */
 public enum Op {
   // identifiers
   ID(true),
@@ -130,6 +130,9 @@ public enum Op {
   CASE,
   FROM,
   SCAN(" "),
+  LEFT_JOIN(" left join "),
+  RIGHT_JOIN(" right join "),
+  FULL_JOIN(" full join "),
   WHERE,
   GROUP,
   COMPUTE,
@@ -196,7 +199,7 @@ public enum Op {
     this.padded = padded;
     this.left = left;
     this.right = right;
-    this.opName = padded == null || padded.equals("")
+    this.opName = padded == null || padded.isEmpty()
         ? null
         : "op " + padded.trim();
   }

diff --git a/src/main/java/net/hydromatic/morel/ast/Shuttle.java b/src/main/java/net/hydromatic/morel/ast/Shuttle.java
@@ -231,8 +231,8 @@ protected AstNode visit(Ast.OrderItem orderItem) {
   }
 
   protected Ast.Scan visit(Ast.Scan scan) {
-    return ast.scan(scan.pos, scan.pat.accept(this),
-        scan.exp.accept(this),
+    return ast.scan(scan.pos, scan.op, scan.pat.accept(this),
+        scan.exp == null ? null : scan.exp.accept(this),
         scan.condition == null ? null : scan.condition.accept(this));
   }
 

diff --git a/src/main/java/net/hydromatic/morel/compile/CalciteCompiler.java b/src/main/java/net/hydromatic/morel/compile/CalciteCompiler.java
@@ -79,6 +79,7 @@
 import java.util.TreeMap;
 import java.util.function.Predicate;
 
+import static net.hydromatic.morel.ast.AstNodes.joinRelType;
 import static net.hydromatic.morel.ast.CoreBuilder.core;
 import static net.hydromatic.morel.util.Ord.forEachIndexed;
 import static net.hydromatic.morel.util.Static.transform;
@@ -405,6 +406,9 @@ private static void harmonizeRowTypes(RelBuilder relBuilder, int inputCount) {
       private RelContext step(RelContext cx, int i, Core.FromStep fromStep) {
         switch (fromStep.op) {
         case SCAN:
+        case LEFT_JOIN:
+        case RIGHT_JOIN:
+        case FULL_JOIN:
           return join(cx, i, (Core.Scan) fromStep);
         case WHERE:
           return where(cx, (Core.Where) fromStep);
@@ -685,15 +689,6 @@ private RelContext join(RelContext cx, int i, Core.Scan scan) {
     return cx;
   }
 
-  private static JoinRelType joinRelType(Op op) {
-    switch (op) {
-    case SCAN:
-      return JoinRelType.INNER;
-    default:
-      throw new AssertionError(op);
-    }
-  }
-
   private RelContext where(RelContext cx, Core.Where where) {
     cx.relBuilder.filter(cx.varList, translate(cx, where.exp));
     return cx;

diff --git a/src/main/java/net/hydromatic/morel/compile/Compiler.java b/src/main/java/net/hydromatic/morel/compile/Compiler.java
@@ -65,6 +65,7 @@
 import java.util.function.Supplier;
 
 import static net.hydromatic.morel.ast.Ast.Direction.DESC;
+import static net.hydromatic.morel.ast.AstNodes.joinRelType;
 import static net.hydromatic.morel.ast.CoreBuilder.core;
 import static net.hydromatic.morel.util.Pair.forEach;
 import static net.hydromatic.morel.util.Static.skip;
@@ -330,11 +331,18 @@ && getOnlyElement(bindings).id.type.equals(elementType)) {
             elementType);
     switch (firstStep.op) {
     case SCAN:
+    case LEFT_JOIN:
+    case RIGHT_JOIN:
+    case FULL_JOIN:
       final Core.Scan scan = (Core.Scan) firstStep;
       final Code code = compile(cx, scan.exp);
       final Code conditionCode = compile(cx, scan.condition);
+      final ImmutableList<Binding> outerBindings =
+          joinRelType(firstStep.op).generatesNullsOnLeft()
+              ? bindings
+              : ImmutableList.of();
       return () -> Codes.scanRowSink(firstStep.op, scan.pat, code,
-          conditionCode, nextFactory.get());
+          conditionCode, outerBindings, nextFactory.get());
 
     case WHERE:
       final Core.Where where = (Core.Where) firstStep;

diff --git a/src/main/java/net/hydromatic/morel/compile/Extents.java b/src/main/java/net/hydromatic/morel/compile/Extents.java
@@ -241,7 +241,7 @@ public static Core.Decl infinitePats(TypeSystem typeSystem,
                           followingSteps, ImmutablePairList.of());
                   for (Core.FromStep step2 : from.steps) {
                     if (step2 == scan) {
-                      fromBuilder.scan(scan.pat, analysis.extentExp,
+                      fromBuilder.scan(scan.op, scan.pat, analysis.extentExp,
                           scan.condition); // TODO
                     } else if (step2 instanceof Core.Where) {
                       fromBuilder.where(

diff --git a/src/main/java/net/hydromatic/morel/compile/Relationalizer.java b/src/main/java/net/hydromatic/morel/compile/Relationalizer.java
@@ -108,7 +108,7 @@ private Core.From toFrom(Core.Exp exp) {
       final List<Binding> bindings = new ArrayList<>();
       Compiles.acceptBinding(typeSystem, id, bindings);
       final Core.Scan scan =
-          core.scan(bindings, id, exp, core.boolLiteral(true));
+          core.scan(Op.SCAN, bindings, id, exp, core.boolLiteral(true));
       return core.from(typeSystem, ImmutableList.of(scan));
     }
   }

diff --git a/src/main/java/net/hydromatic/morel/compile/Resolver.java b/src/main/java/net/hydromatic/morel/compile/Resolver.java
@@ -894,12 +894,15 @@ private Core.Exp run(List<Ast.FromStep> steps) {
         final ListType listType = (ListType) coreExp.type;
         corePat = r.toCore(scan.pat, listType.elementType);
       }
-      final List<Binding> bindings2 = new ArrayList<>(fromBuilder.bindings());
-      Compiles.acceptBinding(typeMap.typeSystem, corePat, bindings2);
-      Core.Exp coreCondition = scan.condition == null
-          ? core.boolLiteral(true)
-          : r.withEnv(bindings2).toCore(scan.condition);
-      fromBuilder.scan(corePat, coreExp, coreCondition);
+      Core.Exp coreCondition;
+      if (scan.condition == null) {
+        coreCondition = core.boolLiteral(true);
+      } else {
+        final List<Binding> bindings2 = new ArrayList<>(fromBuilder.bindings());
+        Compiles.acceptBinding(typeMap.typeSystem, corePat, bindings2);
+        coreCondition = r.withEnv(bindings2).toCore(scan.condition);
+      }
+      fromBuilder.scan(scan.op, corePat, coreExp, coreCondition);
     }
 
     @Override protected void visit(Ast.Where where) {