diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index eab90071ab4..b510d44c1a4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -318,7 +318,7 @@ private boolean isWildcard(final NamedExpression ex) { return false; } final NameSegment expr = ((SchemaPath)ex.getExpr()).getRootSegment(); - return expr.getPath().contains(SchemaPath.DYNAMIC_STAR); + return StarColumnHelper.isStarColumn(expr.getPath()); } private void setupNewSchemaFromInput(RecordBatch incomingBatch) throws SchemaChangeException { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java index 1e8730514ee..cd598eb9891 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnnestPrel.java @@ -51,7 +51,7 @@ public Iterator iterator() { @Override public T accept(PrelVisitor visitor, X value) throws E { - return visitor.visitPrel(this, value); + return visitor.visitUnnest(this, value); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java index 02c4709b96e..4a27ef01cbd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/BasePrelVisitor.java @@ -23,6 +23,7 @@ import org.apache.drill.exec.planner.physical.ProjectPrel; import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.planner.physical.ScreenPrel; +import org.apache.drill.exec.planner.physical.UnnestPrel; import org.apache.drill.exec.planner.physical.WriterPrel; public class BasePrelVisitor implements PrelVisitor { @@ -58,6 +59,11 @@ public RETURN visitWriter(WriterPrel prel, EXTRA value) throws EXCEP { return visitPrel(prel, value); } + @Override + public RETURN visitUnnest(UnnestPrel prel, EXTRA value) throws EXCEP { + return visitPrel(prel, value); + } + @Override public RETURN visitPrel(Prel prel, EXTRA value) throws EXCEP { throw new UnsupportedOperationException(String.format("No visit method defined for prel %s in visitor %s.", prel, this.getClass().getName())); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java index bd81e98cda1..ad692a03d0f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisitor.java @@ -23,6 +23,7 @@ import org.apache.drill.exec.planner.physical.ProjectPrel; import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.planner.physical.ScreenPrel; +import org.apache.drill.exec.planner.physical.UnnestPrel; import org.apache.drill.exec.planner.physical.WriterPrel; @@ -35,6 +36,7 @@ public interface PrelVisitor { public RETURN visitScan(ScanPrel prel, EXTRA value) throws EXCEP; public RETURN visitJoin(JoinPrel prel, EXTRA value) throws EXCEP; public RETURN visitProject(ProjectPrel prel, EXTRA value) throws EXCEP; + public RETURN visitUnnest(UnnestPrel prel, EXTRA value) throws EXCEP; public RETURN visitPrel(Prel prel, EXTRA value) throws EXCEP; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisualizerVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisualizerVisitor.java index 0ee685f622b..705eb77e6f3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisualizerVisitor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/PrelVisualizerVisitor.java @@ -44,7 +44,7 @@ */ public class PrelVisualizerVisitor - implements PrelVisitor { + extends BasePrelVisitor { public static class VisualizationState { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/StarColumnConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/StarColumnConverter.java index 6e860cca14c..fb604596036 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/StarColumnConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/StarColumnConverter.java @@ -24,12 +24,13 @@ import org.apache.calcite.rel.rules.ProjectRemoveRule; import org.apache.drill.exec.planner.StarColumnHelper; -import org.apache.drill.exec.planner.physical.Prel; -import org.apache.drill.exec.planner.physical.ProjectAllowDupPrel; -import org.apache.drill.exec.planner.physical.ProjectPrel; import org.apache.drill.exec.planner.physical.ScanPrel; import org.apache.drill.exec.planner.physical.ScreenPrel; import org.apache.drill.exec.planner.physical.WriterPrel; +import org.apache.drill.exec.planner.physical.ProjectPrel; +import org.apache.drill.exec.planner.physical.ProjectAllowDupPrel; +import org.apache.drill.exec.planner.physical.UnnestPrel; +import org.apache.drill.exec.planner.physical.Prel; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; @@ -189,14 +190,13 @@ public Prel visitPrel(Prel prel, Void value) throws RuntimeException { return (Prel) prel.copy(prel.getTraitSet(), children); } - @Override - public Prel visitScan(ScanPrel scanPrel, Void value) throws RuntimeException { - if (StarColumnHelper.containsStarColumn(scanPrel.getRowType()) && prefixedForStar) { + private Prel convertStarInLeaf(Prel prel, Void value) throws RuntimeException { + if (StarColumnHelper.containsStarColumn(prel.getRowType()) && prefixedForStar) { List exprs = Lists.newArrayList(); - for (RelDataTypeField field : scanPrel.getRowType().getFieldList()) { - RexNode expr = scanPrel.getCluster().getRexBuilder().makeInputRef(field.getType(), field.getIndex()); + for (RelDataTypeField field : prel.getRowType().getFieldList()) { + RexNode expr = prel.getCluster().getRexBuilder().makeInputRef(field.getType(), field.getIndex()); exprs.add(expr); } @@ -204,23 +204,34 @@ public Prel visitScan(ScanPrel scanPrel, Void value) throws RuntimeException { long tableId = tableNumber.getAndIncrement(); - for (String name : scanPrel.getRowType().getFieldNames()) { + for (String name : prel.getRowType().getFieldNames()) { if (StarColumnHelper.isNonPrefixedStarColumn(name)) { fieldNames.add("T" + tableId + StarColumnHelper.PREFIX_DELIMITER + name); // Add prefix to * column. } else { fieldNames.add(name); // Keep regular column as it is. } } - RelDataType rowType = RexUtil.createStructType(scanPrel.getCluster().getTypeFactory(), - exprs, fieldNames, null); + RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), + exprs, fieldNames, null); // insert a PAS. - ProjectPrel proj = new ProjectPrel(scanPrel.getCluster(), scanPrel.getTraitSet(), scanPrel, exprs, rowType); + ProjectPrel proj = new ProjectPrel(prel.getCluster(), prel.getTraitSet(), prel, exprs, rowType); return proj; } else { - return visitPrel(scanPrel, value); + return visitPrel(prel, value); } + + } + + @Override + public Prel visitScan(ScanPrel scanPrel, Void value) throws RuntimeException { + return convertStarInLeaf(scanPrel, value); + } + + @Override + public Prel visitUnnest(UnnestPrel unnestPrel, Void value) throws RuntimeException { + return convertStarInLeaf(unnestPrel, value); } private List makeUniqueNames(List names) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java index 02f611481e6..09a29ec7e45 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/SqlHandlerUtil.java @@ -31,6 +31,7 @@ import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.exec.planner.StarColumnHelper; import org.apache.drill.exec.planner.common.DrillRelOptUtil; import org.apache.drill.exec.planner.logical.DrillRelFactories; import org.apache.drill.exec.store.AbstractSchema; @@ -156,7 +157,7 @@ public static RelNode qualifyPartitionCol(RelNode input, List partitionC .message("Partition column %s is not in the SELECT list of CTAS!", col) .build(logger); } else { - if (SchemaPath.DYNAMIC_STAR.equals(field.getName())) { + if (StarColumnHelper.isStarColumn(field.getName())) { colRefStarNames.add(col); final List operands = Lists.newArrayList(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/FieldSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/FieldSelection.java index f9fad5b806e..665cb4e7a4b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/FieldSelection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/FieldSelection.java @@ -27,6 +27,7 @@ import org.apache.drill.common.expression.SchemaPath; import com.google.common.collect.Maps; +import org.apache.drill.exec.planner.StarColumnHelper; /** @@ -147,7 +148,7 @@ public FieldSelection getChild(String name){ private static boolean containsStar(List columns) { for (SchemaPath expr : columns) { - if (SchemaPath.DYNAMIC_STAR.equals(expr.getRootSegment().getPath())) { + if (StarColumnHelper.isStarColumn(expr.getRootSegment().getPath())) { return true; } } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestLateralPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestLateralPlans.java index 2125bd14e25..c28889f4d96 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestLateralPlans.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/lateraljoin/TestLateralPlans.java @@ -20,12 +20,12 @@ import static org.junit.Assert.assertEquals; import org.apache.drill.PlanTestBase; -import org.apache.drill.test.BaseTestQuery; +import org.apache.drill.exec.work.prepare.PreparedStatementTestBase; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; -public class TestLateralPlans extends BaseTestQuery { +public class TestLateralPlans extends PreparedStatementTestBase { @BeforeClass public static void enableUnnestLateral() throws Exception { @@ -90,8 +90,50 @@ public void testLateralSqlPlainCol() throws Exception { @Test public void testLateralSqlStar() throws Exception { - String Sql = "select * from cp.`lateraljoin/nested-customer.json` t, unnest(t.orders) t2 limit 1"; - test(Sql); + String Sql = "select * from cp.`lateraljoin/nested-customer.json` t, unnest(t.orders) t2 limit 0"; + + testBuilder() + .unOrdered() + .sqlQuery(Sql) + .baselineColumns("c_name", "c_id", "c_phone", "orders", "c_address", "o_id", "o_shop", "o_amount", "items") + .expectsEmptyResultSet() + .go(); + } + + @Test + public void testLateralSqlStar2() throws Exception { + String Sql = "select c.* from cp.`lateraljoin/nested-customer.json` c, unnest(c.orders) o limit 0"; + + testBuilder() + .unOrdered() + .sqlQuery(Sql) + .baselineColumns("c_name", "c_id", "c_phone", "orders", "c_address") + .expectsEmptyResultSet() + .go(); + } + + @Test + public void testLateralSqlStar3() throws Exception { + String Sql = "select o.*, c.* from cp.`lateraljoin/nested-customer.json` c, unnest(c.orders) o limit 0"; + + testBuilder() + .unOrdered() + .sqlQuery(Sql) + .baselineColumns("o_id", "o_shop", "o_amount", "items", "c_name", "c_id", "c_phone", "orders", "c_address") + .expectsEmptyResultSet() + .go(); + } + + @Test + public void testLateralSqlStar4() throws Exception { + String Sql = "select o.* from cp.`lateraljoin/nested-customer.json` c, unnest(c.orders) o limit 0"; + + testBuilder() + .unOrdered() + .sqlQuery(Sql) + .baselineColumns("o_id", "o_shop", "o_amount", "items") + .expectsEmptyResultSet() + .go(); } @Test diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java index b5450e63f91..ec3e6dfde03 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/DrillTestWrapper.java @@ -511,6 +511,15 @@ protected void compareUnorderedResults() throws Exception { addTypeInfoIfMissing(actual.get(0), testBuilder); addToMaterializedResults(actualRecords, actual, loader); + // If actual result record number is 0, + // and the baseline records does not exist, and baselineColumns provided, + // compare actual column number/names with expected columns + if (actualRecords.size() == 0 + && (baselineRecords == null || baselineRecords.size()==0) + && baselineColumns != null) { + checkColumnDef(loader.getSchema()); + } + // If baseline data was not provided to the test builder directly, we must run a query for the baseline, this includes // the cases where the baseline is stored in a file. if (baselineRecords == null) { @@ -716,6 +725,18 @@ public static void addToMaterializedResults(List> materializ } } + public void checkColumnDef(BatchSchema batchSchema) throws Exception{ + assert (batchSchema != null && batchSchema.getFieldCount()==baselineColumns.length); + for (int i=0; i