diff --git a/dinky-client/dinky-client-1.14/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java b/dinky-client/dinky-client-1.14/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java index 5fc8dc24cb..a23820f74b 100644 --- a/dinky-client/dinky-client-1.14/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java +++ b/dinky-client/dinky-client-1.14/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java @@ -42,6 +42,8 @@ public class RelColumnOrigin { private final boolean isDerived; + private boolean isComputedColumn; + /** * Stores the expression for data conversion, * which source table fields are transformed by which expression the target field @@ -63,6 +65,13 @@ public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDer this.transform = transform; } + public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDerived, boolean isComputedColumn) { + this.originTable = originTable; + this.iOriginColumn = iOriginColumn; + this.isDerived = isDerived; + this.isComputedColumn = isComputedColumn; + } + // ~ Methods ---------------------------------------------------------------- /** @@ -94,6 +103,10 @@ public boolean isDerived() { return isDerived; } + public boolean isComputedColumn() { + return isComputedColumn; + } + public String getTransform() { return transform; } diff --git a/dinky-client/dinky-client-1.14/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/dinky-client/dinky-client-1.14/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java index 5c8aae002a..cafe734228 100644 --- a/dinky-client/dinky-client-1.14/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java +++ b/dinky-client/dinky-client-1.14/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java @@ -48,6 +48,8 @@ import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.BuiltInMethod; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.planner.plan.schema.TableSourceTable; import java.util.ArrayList; import java.util.Collections; @@ -230,14 +232,28 @@ public Set getColumnOrigins(Project rel, final RelMetadataQuery // Direct reference: no derivation added. RexInputRef inputRef = (RexInputRef) rexNode; int index = inputRef.getIndex(); - if (input instanceof TableScan) { - index = computeIndexWithOffset(rel.getProjects(), inputRef.getIndex(), iOutputColumn); - } return mq.getColumnOrigins(input, index); } else if (input instanceof TableScan && rexNode.getClass().equals(RexCall.class) && ((RexCall) rexNode).getOperands().isEmpty()) { - return mq.getColumnOrigins(input, iOutputColumn); + List columns = ((TableSourceTable) (input).getTable()) + .catalogTable() + .getResolvedSchema() + .getColumns(); + Set set = new LinkedHashSet<>(); + for (int index = 0; index < columns.size(); index++) { + Column column = columns.get(index); + if (column instanceof Column.ComputedColumn + && rexNode.toString() + .equals(((Column.ComputedColumn) column) + .getExpression() + .toString())) { + set.add(new RelColumnOrigin(input.getTable(), index, false, true)); + return set; + } + } + set.add(new RelColumnOrigin(input.getTable(), -1, false, false)); + return set; } // Anything else is a derivation, possibly from multiple columns. final Set set = getMultipleColumns(rexNode, input, mq); diff --git a/dinky-client/dinky-client-1.14/src/main/java/org/dinky/utils/LineageContext.java b/dinky-client/dinky-client-1.14/src/main/java/org/dinky/utils/LineageContext.java index daf07362b8..f9c262ac31 100644 --- a/dinky-client/dinky-client-1.14/src/main/java/org/dinky/utils/LineageContext.java +++ b/dinky-client/dinky-client-1.14/src/main/java/org/dinky/utils/LineageContext.java @@ -128,11 +128,22 @@ private List buildFiledLineageResult(String sinkTable, RelNode optRe // filed int ordinal = relColumnOrigin.getOriginColumnOrdinal(); - List fieldNames = ((TableSourceTable) table) - .catalogTable() - .getResolvedSchema() - .getColumnNames(); - String sourceColumn = fieldNames.get(ordinal); + + if (ordinal == -1) { + continue; + } + + String sourceColumn; + if (relColumnOrigin.isComputedColumn()) { + List fieldNames = ((TableSourceTable) table) + .catalogTable() + .getResolvedSchema() + .getColumnNames(); + sourceColumn = fieldNames.get(ordinal); + } else { + List fieldNames = table.getRowType().getFieldNames(); + sourceColumn = fieldNames.get(ordinal); + } // add record resultList.add(LineageRel.build( diff --git a/dinky-client/dinky-client-1.15/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java b/dinky-client/dinky-client-1.15/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java index 5fc8dc24cb..86ab256fdf 100644 --- a/dinky-client/dinky-client-1.15/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java +++ b/dinky-client/dinky-client-1.15/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java @@ -42,6 +42,8 @@ public class RelColumnOrigin { private final boolean isDerived; + private boolean isComputedColumn; + /** * Stores the expression for data conversion, * which source table fields are transformed by which expression the target field @@ -63,6 +65,13 @@ public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDer this.transform = transform; } + public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDerived, boolean isComputedColumn) { + this.originTable = originTable; + this.iOriginColumn = iOriginColumn; + this.isDerived = isDerived; + this.isComputedColumn = isComputedColumn; + } + // ~ Methods ---------------------------------------------------------------- /** @@ -98,6 +107,10 @@ public String getTransform() { return transform; } + public boolean isComputedColumn() { + return isComputedColumn; + } + @Override public boolean equals(Object obj) { if (!(obj instanceof RelColumnOrigin)) { diff --git a/dinky-client/dinky-client-1.15/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/dinky-client/dinky-client-1.15/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java index 5c8aae002a..176b59a87f 100644 --- a/dinky-client/dinky-client-1.15/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java +++ b/dinky-client/dinky-client-1.15/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java @@ -48,6 +48,8 @@ import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.BuiltInMethod; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.planner.plan.schema.TableSourceTable; import java.util.ArrayList; import java.util.Collections; @@ -230,14 +232,28 @@ public Set getColumnOrigins(Project rel, final RelMetadataQuery // Direct reference: no derivation added. RexInputRef inputRef = (RexInputRef) rexNode; int index = inputRef.getIndex(); - if (input instanceof TableScan) { - index = computeIndexWithOffset(rel.getProjects(), inputRef.getIndex(), iOutputColumn); - } return mq.getColumnOrigins(input, index); } else if (input instanceof TableScan && rexNode.getClass().equals(RexCall.class) && ((RexCall) rexNode).getOperands().isEmpty()) { - return mq.getColumnOrigins(input, iOutputColumn); + List columns = ((TableSourceTable) (input).getTable()) + .contextResolvedTable() + .getResolvedSchema() + .getColumns(); + Set set = new LinkedHashSet<>(); + for (int index = 0; index < columns.size(); index++) { + Column column = columns.get(index); + if (column instanceof Column.ComputedColumn + && rexNode.toString() + .equals(((Column.ComputedColumn) column) + .getExpression() + .toString())) { + set.add(new RelColumnOrigin(input.getTable(), index, false, true)); + return set; + } + } + set.add(new RelColumnOrigin(input.getTable(), -1, false, false)); + return set; } // Anything else is a derivation, possibly from multiple columns. final Set set = getMultipleColumns(rexNode, input, mq); diff --git a/dinky-client/dinky-client-1.15/src/main/java/org/dinky/utils/LineageContext.java b/dinky-client/dinky-client-1.15/src/main/java/org/dinky/utils/LineageContext.java index 059cbf0d71..23e18d01d2 100644 --- a/dinky-client/dinky-client-1.15/src/main/java/org/dinky/utils/LineageContext.java +++ b/dinky-client/dinky-client-1.15/src/main/java/org/dinky/utils/LineageContext.java @@ -131,11 +131,22 @@ private List buildFiledLineageResult(String sinkTable, RelNode optRe // filed int ordinal = relColumnOrigin.getOriginColumnOrdinal(); - List fieldNames = ((TableSourceTable) table) - .contextResolvedTable() - .getResolvedSchema() - .getColumnNames(); - String sourceColumn = fieldNames.get(ordinal); + + if (ordinal == -1) { + continue; + } + + String sourceColumn; + if (relColumnOrigin.isComputedColumn()) { + List fieldNames = ((TableSourceTable) table) + .contextResolvedTable() + .getResolvedSchema() + .getColumnNames(); + sourceColumn = fieldNames.get(ordinal); + } else { + List fieldNames = table.getRowType().getFieldNames(); + sourceColumn = fieldNames.get(ordinal); + } // add record resultList.add(LineageRel.build( diff --git a/dinky-client/dinky-client-1.16/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java b/dinky-client/dinky-client-1.16/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java index 5fc8dc24cb..86ab256fdf 100644 --- a/dinky-client/dinky-client-1.16/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java +++ b/dinky-client/dinky-client-1.16/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java @@ -42,6 +42,8 @@ public class RelColumnOrigin { private final boolean isDerived; + private boolean isComputedColumn; + /** * Stores the expression for data conversion, * which source table fields are transformed by which expression the target field @@ -63,6 +65,13 @@ public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDer this.transform = transform; } + public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDerived, boolean isComputedColumn) { + this.originTable = originTable; + this.iOriginColumn = iOriginColumn; + this.isDerived = isDerived; + this.isComputedColumn = isComputedColumn; + } + // ~ Methods ---------------------------------------------------------------- /** @@ -98,6 +107,10 @@ public String getTransform() { return transform; } + public boolean isComputedColumn() { + return isComputedColumn; + } + @Override public boolean equals(Object obj) { if (!(obj instanceof RelColumnOrigin)) { diff --git a/dinky-client/dinky-client-1.16/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/dinky-client/dinky-client-1.16/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java index 5c8aae002a..b3d7471cd0 100644 --- a/dinky-client/dinky-client-1.16/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java +++ b/dinky-client/dinky-client-1.16/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java @@ -48,6 +48,8 @@ import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.BuiltInMethod; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.planner.plan.schema.TableSourceTable; import java.util.ArrayList; import java.util.Collections; @@ -230,14 +232,31 @@ public Set getColumnOrigins(Project rel, final RelMetadataQuery // Direct reference: no derivation added. RexInputRef inputRef = (RexInputRef) rexNode; int index = inputRef.getIndex(); - if (input instanceof TableScan) { - index = computeIndexWithOffset(rel.getProjects(), inputRef.getIndex(), iOutputColumn); - } + // if (input instanceof TableScan) { + // index = computeIndexWithOffset(rel.getProjects(), inputRef.getIndex(), iOutputColumn); + // } return mq.getColumnOrigins(input, index); } else if (input instanceof TableScan && rexNode.getClass().equals(RexCall.class) && ((RexCall) rexNode).getOperands().isEmpty()) { - return mq.getColumnOrigins(input, iOutputColumn); + List columns = ((TableSourceTable) (input).getTable()) + .contextResolvedTable() + .getResolvedSchema() + .getColumns(); + Set set = new LinkedHashSet<>(); + for (int index = 0; index < columns.size(); index++) { + Column column = columns.get(index); + if (column instanceof Column.ComputedColumn + && rexNode.toString() + .equals(((Column.ComputedColumn) column) + .getExpression() + .toString())) { + set.add(new RelColumnOrigin(input.getTable(), index, false, true)); + return set; + } + } + set.add(new RelColumnOrigin(input.getTable(), -1, false, false)); + return set; } // Anything else is a derivation, possibly from multiple columns. final Set set = getMultipleColumns(rexNode, input, mq); diff --git a/dinky-client/dinky-client-1.16/src/main/java/org/dinky/utils/LineageContext.java b/dinky-client/dinky-client-1.16/src/main/java/org/dinky/utils/LineageContext.java index 455f1df54a..d2586f18d3 100644 --- a/dinky-client/dinky-client-1.16/src/main/java/org/dinky/utils/LineageContext.java +++ b/dinky-client/dinky-client-1.16/src/main/java/org/dinky/utils/LineageContext.java @@ -133,11 +133,22 @@ private List buildFiledLineageResult(String sinkTable, RelNode optRe // filed int ordinal = relColumnOrigin.getOriginColumnOrdinal(); - List fieldNames = ((TableSourceTable) table) - .contextResolvedTable() - .getResolvedSchema() - .getColumnNames(); - String sourceColumn = fieldNames.get(ordinal); + + if (ordinal == -1) { + continue; + } + + String sourceColumn; + if (relColumnOrigin.isComputedColumn()) { + List fieldNames = ((TableSourceTable) table) + .contextResolvedTable() + .getResolvedSchema() + .getColumnNames(); + sourceColumn = fieldNames.get(ordinal); + } else { + List fieldNames = table.getRowType().getFieldNames(); + sourceColumn = fieldNames.get(ordinal); + } // add record resultList.add(LineageRel.build( diff --git a/dinky-client/dinky-client-1.17/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java b/dinky-client/dinky-client-1.17/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java index 5fc8dc24cb..86ab256fdf 100644 --- a/dinky-client/dinky-client-1.17/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java +++ b/dinky-client/dinky-client-1.17/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java @@ -42,6 +42,8 @@ public class RelColumnOrigin { private final boolean isDerived; + private boolean isComputedColumn; + /** * Stores the expression for data conversion, * which source table fields are transformed by which expression the target field @@ -63,6 +65,13 @@ public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDer this.transform = transform; } + public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDerived, boolean isComputedColumn) { + this.originTable = originTable; + this.iOriginColumn = iOriginColumn; + this.isDerived = isDerived; + this.isComputedColumn = isComputedColumn; + } + // ~ Methods ---------------------------------------------------------------- /** @@ -98,6 +107,10 @@ public String getTransform() { return transform; } + public boolean isComputedColumn() { + return isComputedColumn; + } + @Override public boolean equals(Object obj) { if (!(obj instanceof RelColumnOrigin)) { diff --git a/dinky-client/dinky-client-1.17/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/dinky-client/dinky-client-1.17/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java index 5c8aae002a..176b59a87f 100644 --- a/dinky-client/dinky-client-1.17/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java +++ b/dinky-client/dinky-client-1.17/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java @@ -48,6 +48,8 @@ import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.BuiltInMethod; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.planner.plan.schema.TableSourceTable; import java.util.ArrayList; import java.util.Collections; @@ -230,14 +232,28 @@ public Set getColumnOrigins(Project rel, final RelMetadataQuery // Direct reference: no derivation added. RexInputRef inputRef = (RexInputRef) rexNode; int index = inputRef.getIndex(); - if (input instanceof TableScan) { - index = computeIndexWithOffset(rel.getProjects(), inputRef.getIndex(), iOutputColumn); - } return mq.getColumnOrigins(input, index); } else if (input instanceof TableScan && rexNode.getClass().equals(RexCall.class) && ((RexCall) rexNode).getOperands().isEmpty()) { - return mq.getColumnOrigins(input, iOutputColumn); + List columns = ((TableSourceTable) (input).getTable()) + .contextResolvedTable() + .getResolvedSchema() + .getColumns(); + Set set = new LinkedHashSet<>(); + for (int index = 0; index < columns.size(); index++) { + Column column = columns.get(index); + if (column instanceof Column.ComputedColumn + && rexNode.toString() + .equals(((Column.ComputedColumn) column) + .getExpression() + .toString())) { + set.add(new RelColumnOrigin(input.getTable(), index, false, true)); + return set; + } + } + set.add(new RelColumnOrigin(input.getTable(), -1, false, false)); + return set; } // Anything else is a derivation, possibly from multiple columns. final Set set = getMultipleColumns(rexNode, input, mq); diff --git a/dinky-client/dinky-client-1.17/src/main/java/org/dinky/utils/LineageContext.java b/dinky-client/dinky-client-1.17/src/main/java/org/dinky/utils/LineageContext.java index c924b9c86d..c40bdab84c 100644 --- a/dinky-client/dinky-client-1.17/src/main/java/org/dinky/utils/LineageContext.java +++ b/dinky-client/dinky-client-1.17/src/main/java/org/dinky/utils/LineageContext.java @@ -131,11 +131,22 @@ private List buildFiledLineageResult(String sinkTable, RelNode optRe // filed int ordinal = relColumnOrigin.getOriginColumnOrdinal(); - List fieldNames = ((TableSourceTable) table) - .contextResolvedTable() - .getResolvedSchema() - .getColumnNames(); - String sourceColumn = fieldNames.get(ordinal); + + if (ordinal == -1) { + continue; + } + + String sourceColumn; + if (relColumnOrigin.isComputedColumn()) { + List fieldNames = ((TableSourceTable) table) + .contextResolvedTable() + .getResolvedSchema() + .getColumnNames(); + sourceColumn = fieldNames.get(ordinal); + } else { + List fieldNames = table.getRowType().getFieldNames(); + sourceColumn = fieldNames.get(ordinal); + } // add record resultList.add(LineageRel.build( diff --git a/dinky-client/dinky-client-1.18/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java b/dinky-client/dinky-client-1.18/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java index 5fc8dc24cb..86ab256fdf 100644 --- a/dinky-client/dinky-client-1.18/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java +++ b/dinky-client/dinky-client-1.18/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java @@ -42,6 +42,8 @@ public class RelColumnOrigin { private final boolean isDerived; + private boolean isComputedColumn; + /** * Stores the expression for data conversion, * which source table fields are transformed by which expression the target field @@ -63,6 +65,13 @@ public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDer this.transform = transform; } + public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDerived, boolean isComputedColumn) { + this.originTable = originTable; + this.iOriginColumn = iOriginColumn; + this.isDerived = isDerived; + this.isComputedColumn = isComputedColumn; + } + // ~ Methods ---------------------------------------------------------------- /** @@ -98,6 +107,10 @@ public String getTransform() { return transform; } + public boolean isComputedColumn() { + return isComputedColumn; + } + @Override public boolean equals(Object obj) { if (!(obj instanceof RelColumnOrigin)) { diff --git a/dinky-client/dinky-client-1.18/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/dinky-client/dinky-client-1.18/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java index 5c8aae002a..176b59a87f 100644 --- a/dinky-client/dinky-client-1.18/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java +++ b/dinky-client/dinky-client-1.18/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java @@ -48,6 +48,8 @@ import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.BuiltInMethod; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.planner.plan.schema.TableSourceTable; import java.util.ArrayList; import java.util.Collections; @@ -230,14 +232,28 @@ public Set getColumnOrigins(Project rel, final RelMetadataQuery // Direct reference: no derivation added. RexInputRef inputRef = (RexInputRef) rexNode; int index = inputRef.getIndex(); - if (input instanceof TableScan) { - index = computeIndexWithOffset(rel.getProjects(), inputRef.getIndex(), iOutputColumn); - } return mq.getColumnOrigins(input, index); } else if (input instanceof TableScan && rexNode.getClass().equals(RexCall.class) && ((RexCall) rexNode).getOperands().isEmpty()) { - return mq.getColumnOrigins(input, iOutputColumn); + List columns = ((TableSourceTable) (input).getTable()) + .contextResolvedTable() + .getResolvedSchema() + .getColumns(); + Set set = new LinkedHashSet<>(); + for (int index = 0; index < columns.size(); index++) { + Column column = columns.get(index); + if (column instanceof Column.ComputedColumn + && rexNode.toString() + .equals(((Column.ComputedColumn) column) + .getExpression() + .toString())) { + set.add(new RelColumnOrigin(input.getTable(), index, false, true)); + return set; + } + } + set.add(new RelColumnOrigin(input.getTable(), -1, false, false)); + return set; } // Anything else is a derivation, possibly from multiple columns. final Set set = getMultipleColumns(rexNode, input, mq); diff --git a/dinky-client/dinky-client-1.18/src/main/java/org/dinky/utils/LineageContext.java b/dinky-client/dinky-client-1.18/src/main/java/org/dinky/utils/LineageContext.java index c924b9c86d..c40bdab84c 100644 --- a/dinky-client/dinky-client-1.18/src/main/java/org/dinky/utils/LineageContext.java +++ b/dinky-client/dinky-client-1.18/src/main/java/org/dinky/utils/LineageContext.java @@ -131,11 +131,22 @@ private List buildFiledLineageResult(String sinkTable, RelNode optRe // filed int ordinal = relColumnOrigin.getOriginColumnOrdinal(); - List fieldNames = ((TableSourceTable) table) - .contextResolvedTable() - .getResolvedSchema() - .getColumnNames(); - String sourceColumn = fieldNames.get(ordinal); + + if (ordinal == -1) { + continue; + } + + String sourceColumn; + if (relColumnOrigin.isComputedColumn()) { + List fieldNames = ((TableSourceTable) table) + .contextResolvedTable() + .getResolvedSchema() + .getColumnNames(); + sourceColumn = fieldNames.get(ordinal); + } else { + List fieldNames = table.getRowType().getFieldNames(); + sourceColumn = fieldNames.get(ordinal); + } // add record resultList.add(LineageRel.build( diff --git a/dinky-client/dinky-client-1.19/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java b/dinky-client/dinky-client-1.19/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java index 5fc8dc24cb..86ab256fdf 100644 --- a/dinky-client/dinky-client-1.19/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java +++ b/dinky-client/dinky-client-1.19/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java @@ -42,6 +42,8 @@ public class RelColumnOrigin { private final boolean isDerived; + private boolean isComputedColumn; + /** * Stores the expression for data conversion, * which source table fields are transformed by which expression the target field @@ -63,6 +65,13 @@ public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDer this.transform = transform; } + public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDerived, boolean isComputedColumn) { + this.originTable = originTable; + this.iOriginColumn = iOriginColumn; + this.isDerived = isDerived; + this.isComputedColumn = isComputedColumn; + } + // ~ Methods ---------------------------------------------------------------- /** @@ -98,6 +107,10 @@ public String getTransform() { return transform; } + public boolean isComputedColumn() { + return isComputedColumn; + } + @Override public boolean equals(Object obj) { if (!(obj instanceof RelColumnOrigin)) { diff --git a/dinky-client/dinky-client-1.19/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/dinky-client/dinky-client-1.19/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java index 5c8aae002a..176b59a87f 100644 --- a/dinky-client/dinky-client-1.19/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java +++ b/dinky-client/dinky-client-1.19/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java @@ -48,6 +48,8 @@ import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.BuiltInMethod; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.planner.plan.schema.TableSourceTable; import java.util.ArrayList; import java.util.Collections; @@ -230,14 +232,28 @@ public Set getColumnOrigins(Project rel, final RelMetadataQuery // Direct reference: no derivation added. RexInputRef inputRef = (RexInputRef) rexNode; int index = inputRef.getIndex(); - if (input instanceof TableScan) { - index = computeIndexWithOffset(rel.getProjects(), inputRef.getIndex(), iOutputColumn); - } return mq.getColumnOrigins(input, index); } else if (input instanceof TableScan && rexNode.getClass().equals(RexCall.class) && ((RexCall) rexNode).getOperands().isEmpty()) { - return mq.getColumnOrigins(input, iOutputColumn); + List columns = ((TableSourceTable) (input).getTable()) + .contextResolvedTable() + .getResolvedSchema() + .getColumns(); + Set set = new LinkedHashSet<>(); + for (int index = 0; index < columns.size(); index++) { + Column column = columns.get(index); + if (column instanceof Column.ComputedColumn + && rexNode.toString() + .equals(((Column.ComputedColumn) column) + .getExpression() + .toString())) { + set.add(new RelColumnOrigin(input.getTable(), index, false, true)); + return set; + } + } + set.add(new RelColumnOrigin(input.getTable(), -1, false, false)); + return set; } // Anything else is a derivation, possibly from multiple columns. final Set set = getMultipleColumns(rexNode, input, mq); diff --git a/dinky-client/dinky-client-1.19/src/main/java/org/dinky/utils/LineageContext.java b/dinky-client/dinky-client-1.19/src/main/java/org/dinky/utils/LineageContext.java index c924b9c86d..c40bdab84c 100644 --- a/dinky-client/dinky-client-1.19/src/main/java/org/dinky/utils/LineageContext.java +++ b/dinky-client/dinky-client-1.19/src/main/java/org/dinky/utils/LineageContext.java @@ -131,11 +131,22 @@ private List buildFiledLineageResult(String sinkTable, RelNode optRe // filed int ordinal = relColumnOrigin.getOriginColumnOrdinal(); - List fieldNames = ((TableSourceTable) table) - .contextResolvedTable() - .getResolvedSchema() - .getColumnNames(); - String sourceColumn = fieldNames.get(ordinal); + + if (ordinal == -1) { + continue; + } + + String sourceColumn; + if (relColumnOrigin.isComputedColumn()) { + List fieldNames = ((TableSourceTable) table) + .contextResolvedTable() + .getResolvedSchema() + .getColumnNames(); + sourceColumn = fieldNames.get(ordinal); + } else { + List fieldNames = table.getRowType().getFieldNames(); + sourceColumn = fieldNames.get(ordinal); + } // add record resultList.add(LineageRel.build( diff --git a/dinky-client/dinky-client-1.20/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java b/dinky-client/dinky-client-1.20/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java index 5fc8dc24cb..86ab256fdf 100644 --- a/dinky-client/dinky-client-1.20/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java +++ b/dinky-client/dinky-client-1.20/src/main/java/org/apache/calcite/rel/metadata/RelColumnOrigin.java @@ -42,6 +42,8 @@ public class RelColumnOrigin { private final boolean isDerived; + private boolean isComputedColumn; + /** * Stores the expression for data conversion, * which source table fields are transformed by which expression the target field @@ -63,6 +65,13 @@ public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDer this.transform = transform; } + public RelColumnOrigin(RelOptTable originTable, int iOriginColumn, boolean isDerived, boolean isComputedColumn) { + this.originTable = originTable; + this.iOriginColumn = iOriginColumn; + this.isDerived = isDerived; + this.isComputedColumn = isComputedColumn; + } + // ~ Methods ---------------------------------------------------------------- /** @@ -98,6 +107,10 @@ public String getTransform() { return transform; } + public boolean isComputedColumn() { + return isComputedColumn; + } + @Override public boolean equals(Object obj) { if (!(obj instanceof RelColumnOrigin)) { diff --git a/dinky-client/dinky-client-1.20/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java b/dinky-client/dinky-client-1.20/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java index 5c8aae002a..176b59a87f 100644 --- a/dinky-client/dinky-client-1.20/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java +++ b/dinky-client/dinky-client-1.20/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnOrigins.java @@ -48,6 +48,8 @@ import org.apache.calcite.rex.RexVisitor; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.util.BuiltInMethod; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.planner.plan.schema.TableSourceTable; import java.util.ArrayList; import java.util.Collections; @@ -230,14 +232,28 @@ public Set getColumnOrigins(Project rel, final RelMetadataQuery // Direct reference: no derivation added. RexInputRef inputRef = (RexInputRef) rexNode; int index = inputRef.getIndex(); - if (input instanceof TableScan) { - index = computeIndexWithOffset(rel.getProjects(), inputRef.getIndex(), iOutputColumn); - } return mq.getColumnOrigins(input, index); } else if (input instanceof TableScan && rexNode.getClass().equals(RexCall.class) && ((RexCall) rexNode).getOperands().isEmpty()) { - return mq.getColumnOrigins(input, iOutputColumn); + List columns = ((TableSourceTable) (input).getTable()) + .contextResolvedTable() + .getResolvedSchema() + .getColumns(); + Set set = new LinkedHashSet<>(); + for (int index = 0; index < columns.size(); index++) { + Column column = columns.get(index); + if (column instanceof Column.ComputedColumn + && rexNode.toString() + .equals(((Column.ComputedColumn) column) + .getExpression() + .toString())) { + set.add(new RelColumnOrigin(input.getTable(), index, false, true)); + return set; + } + } + set.add(new RelColumnOrigin(input.getTable(), -1, false, false)); + return set; } // Anything else is a derivation, possibly from multiple columns. final Set set = getMultipleColumns(rexNode, input, mq); diff --git a/dinky-client/dinky-client-1.20/src/main/java/org/dinky/utils/LineageContext.java b/dinky-client/dinky-client-1.20/src/main/java/org/dinky/utils/LineageContext.java index c924b9c86d..c40bdab84c 100644 --- a/dinky-client/dinky-client-1.20/src/main/java/org/dinky/utils/LineageContext.java +++ b/dinky-client/dinky-client-1.20/src/main/java/org/dinky/utils/LineageContext.java @@ -131,11 +131,22 @@ private List buildFiledLineageResult(String sinkTable, RelNode optRe // filed int ordinal = relColumnOrigin.getOriginColumnOrdinal(); - List fieldNames = ((TableSourceTable) table) - .contextResolvedTable() - .getResolvedSchema() - .getColumnNames(); - String sourceColumn = fieldNames.get(ordinal); + + if (ordinal == -1) { + continue; + } + + String sourceColumn; + if (relColumnOrigin.isComputedColumn()) { + List fieldNames = ((TableSourceTable) table) + .contextResolvedTable() + .getResolvedSchema() + .getColumnNames(); + sourceColumn = fieldNames.get(ordinal); + } else { + List fieldNames = table.getRowType().getFieldNames(); + sourceColumn = fieldNames.get(ordinal); + } // add record resultList.add(LineageRel.build( diff --git a/dinky-core/src/main/java/org/dinky/explainer/Explainer.java b/dinky-core/src/main/java/org/dinky/explainer/Explainer.java index 18065b84a2..1b266f4858 100644 --- a/dinky-core/src/main/java/org/dinky/explainer/Explainer.java +++ b/dinky-core/src/main/java/org/dinky/explainer/Explainer.java @@ -21,6 +21,7 @@ import org.dinky.assertion.Asserts; import org.dinky.data.enums.GatewayType; +import org.dinky.data.exception.DinkyException; import org.dinky.data.model.LineageRel; import org.dinky.data.result.ExplainResult; import org.dinky.data.result.SqlExplainResult; @@ -415,8 +416,8 @@ public List getLineage(String statement) { executor.executeSql(sql); } } catch (Exception e) { - log.error(e.getMessage()); - return lineageRelList; + log.error("Exception occurred while fetching lineage information", e); + throw new DinkyException("Exception occurred while fetching lineage information", e); } } return lineageRelList;