SOLR-15461: Upgrade Apache Calcite to latest release. (apache#177)

solr/CHANGES.txt

@@ -335,6 +335,8 @@ Other Changes
   collections/collectionName into its state.json.  For many-collection clusters, this is an
   optimization when the cluster status is fetched.  (Nazerke Seidan, David Smiley)
 
+* SOLR-15461: SOLR-15461: Upgrade Apache Calcite to 1.27.0. (Mark Miller, Timothy Potter)
+
 Bug Fixes
 ---------------------
 * SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order execution

solr/core/build.gradle

@@ -102,6 +102,9 @@ dependencies {
 
   implementation ('org.apache.calcite.avatica:avatica-core') { transitive = false }
   implementation ('org.apache.calcite:calcite-core') { transitive = false }
+  implementation ('org.checkerframework:checker-qual') { transitive = false }
+  implementation ('com.esri.geometry:esri-geometry-api') { transitive = false }
+  implementation ('org.apiguardian:apiguardian-api')  { transitive = false }
   implementation ('org.apache.calcite:calcite-linq4j') { transitive = false }
   implementation ('org.apache.curator:curator-client') { transitive = false }
   implementation ('org.apache.curator:curator-framework') { transitive = false }

solr/core/src/java/org/apache/solr/handler/sql/SolrAggregate.java

@@ -21,6 +21,7 @@
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Aggregate;
 import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.hint.RelHint;
 import org.apache.calcite.sql.SqlAggFunction;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.util.ImmutableBitSet;
@@ -44,21 +45,27 @@ class SolrAggregate extends Aggregate implements SolrRel {
   SolrAggregate(
       RelOptCluster cluster,
       RelTraitSet traitSet,
+      List<RelHint> hints,
       RelNode child,
-      boolean indicator,
       ImmutableBitSet groupSet,
       List<ImmutableBitSet> groupSets,
       List<AggregateCall> aggCalls) {
-    super(cluster, traitSet, child, indicator, groupSet, groupSets, aggCalls);
+    super(cluster, traitSet, hints, child, groupSet, groupSets, aggCalls);
     assert getConvention() == SolrRel.CONVENTION;
     assert getConvention() == child.getConvention();
   }
 
+  @Override
+  public Aggregate copy(RelTraitSet traitSet, RelNode input, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets,
+      List<AggregateCall> aggCalls) {
+    return new SolrAggregate(getCluster(), traitSet, hints,input, groupSet, groupSets, aggCalls);
+  }
+
   @Override
   public Aggregate copy(RelTraitSet traitSet, RelNode input,
                         boolean indicator, ImmutableBitSet groupSet,
                         List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
-    return new SolrAggregate(getCluster(), traitSet, input, indicator, groupSet, groupSets, aggCalls);
+    return new SolrAggregate(getCluster(), traitSet, hints, input, groupSet, groupSets, aggCalls);
   }
 
   public void implement(Implementor implementor) {

solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java

@@ -16,6 +16,7 @@
  */
 package org.apache.solr.handler.sql;
 
+import java.lang.invoke.MethodHandles;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
@@ -30,20 +31,22 @@
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.*;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Implementation of a {@link org.apache.calcite.rel.core.Filter} relational expression in Solr.
  */
 class SolrFilter extends Filter implements SolrRel {
 
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
   private static final Pattern CALCITE_TIMESTAMP_REGEX = Pattern.compile("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$");
+  private final RexBuilder builder;
 
   SolrFilter(
       RelOptCluster cluster,
@@ -53,6 +56,7 @@ class SolrFilter extends Filter implements SolrRel {
     super(cluster, traitSet, child, condition);
     assert getConvention() == SolrRel.CONVENTION;
     assert getConvention() == child.getConvention();
+    builder = child.getCluster().getRexBuilder();
   }
 
   @Override
@@ -67,11 +71,11 @@ public SolrFilter copy(RelTraitSet traitSet, RelNode input, RexNode condition) {
   public void implement(Implementor implementor) {
     implementor.visitChild(0, getInput());
     if (getInput() instanceof SolrAggregate) {
-      HavingTranslator translator = new HavingTranslator(SolrRules.solrFieldNames(getRowType()), implementor.reverseAggMappings);
+      HavingTranslator translator = new HavingTranslator(SolrRules.solrFieldNames(getRowType()), implementor.reverseAggMappings, builder);
       String havingPredicate = translator.translateMatch(condition);
       implementor.setHavingPredicate(havingPredicate);
     } else {
-      Translator translator = new Translator(SolrRules.solrFieldNames(getRowType()));
+      Translator translator = new Translator(SolrRules.solrFieldNames(getRowType()), builder);
       String query = translator.translateMatch(condition);
       implementor.addQuery(query);
       implementor.setNegativeQuery(translator.negativeQuery);
@@ -81,14 +85,26 @@ public void implement(Implementor implementor) {
   private static class Translator {
 
     protected final List<String> fieldNames;
+    private final RexBuilder builder;
     public boolean negativeQuery = true;
 
-    Translator(List<String> fieldNames) {
+    Translator(List<String> fieldNames, RexBuilder builder) {
       this.fieldNames = fieldNames;
+      this.builder = builder;
     }
 
     protected String translateMatch(RexNode condition) {
+      if (log.isDebugEnabled()) {
+        log.debug("translateMatch condition={} {}", condition.getKind(), condition.getClass().getName());
+      }
+
       final SqlKind kind = condition.getKind();
+
+      if (condition.isA(SqlKind.SEARCH)) {
+
+        return translateMatch(RexUtil.expandSearch(builder, null, condition));
+      }
+
       if (kind.belongsTo(SqlKind.COMPARISON) || kind == SqlKind.NOT) {
         return translateComparison(condition);
       } else if (condition.isA(SqlKind.AND)) {
@@ -107,7 +123,11 @@ protected String translateMatch(RexNode condition) {
             query = translateBetween(rhs, lhs);
           }
         }
-        return query != null ? query : "(" + translateAnd(condition) + ")";
+        query = query != null ? query : "(" + translateAnd(condition) + ")";
+        if (log.isDebugEnabled()) {
+          log.debug("translated query match={}", query);
+        }
+        return query;
       } else if (condition.isA(SqlKind.OR)) {
         return "(" + translateOr(condition) + ")";
       } else if (kind == SqlKind.LIKE) {
@@ -323,14 +343,25 @@ protected Pair<String, RexLiteral> translateBinary(RexCall call) {
       if (b != null) {
         return b;
       }
+
+      if (left.getKind() == SqlKind.CAST && right.getKind() == SqlKind.CAST) {
+        return translateBinary2(((RexCall)left).operands.get(0), ((RexCall)right).operands.get(0));
+      }
+
       throw new AssertionError("cannot translate call " + call);
     }
 
     /**
      * Translates a call to a binary operator. Returns whether successful.
      */
     protected Pair<String, RexLiteral> translateBinary2(RexNode left, RexNode right) {
+      if (log.isDebugEnabled()) {
+        log.debug("translateBinary2 left={} right={}", left, right);
+      }
       if (right.getKind() != SqlKind.LITERAL) {
+        if (log.isDebugEnabled()) {
+          log.debug("right != SqlKind.LITERAL, return null");
+        }
         return null;
       }
 
@@ -357,8 +388,8 @@ private static class HavingTranslator extends Translator {
 
     private final Map<String, String> reverseAggMappings;
 
-    HavingTranslator(List<String> fieldNames, Map<String, String> reverseAggMappings) {
-      super(fieldNames);
+    HavingTranslator(List<String> fieldNames, Map<String, String> reverseAggMappings, RexBuilder builder) {
+      super(fieldNames, builder);
       this.reverseAggMappings = reverseAggMappings;
     }
 
@@ -474,7 +505,7 @@ protected String translateComparison(RexNode node) {
       Pair<String, RexLiteral> binaryTranslated = getFieldValuePair(node);
       switch (node.getKind()) {
         case EQUALS:
-          String terms = binaryTranslated.getValue().toString().trim();
+          String terms = binaryTranslated.getValue().getValue2().toString().trim();
           return "eq(" + binaryTranslated.getKey() + "," + terms + ")";
         case NOT_EQUALS:
           return "not(eq(" + binaryTranslated.getKey() + "," + binaryTranslated.getValue() + "))";

solr/core/src/java/org/apache/solr/handler/sql/SolrRules.java

@@ -26,9 +26,6 @@
 import org.apache.calcite.rel.logical.LogicalFilter;
 import org.apache.calcite.rel.logical.LogicalProject;
 import org.apache.calcite.rel.logical.LogicalSort;
-import org.apache.calcite.rel.rules.AggregateValuesRule;
-import org.apache.calcite.rel.rules.ReduceExpressionsRule;
-import org.apache.calcite.rel.rules.ValuesReduceRule;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexInputRef;
@@ -55,17 +52,6 @@ class SolrRules {
       SolrAggregateRule.AGGREGATE_RULE,
   };
 
-  static final RelOptRule[] CONSTANT_REDUCTION_RULES = {
-      ReduceExpressionsRule.PROJECT_INSTANCE,
-      ReduceExpressionsRule.FILTER_INSTANCE,
-      ReduceExpressionsRule.CALC_INSTANCE,
-      ReduceExpressionsRule.JOIN_INSTANCE,
-      ValuesReduceRule.FILTER_INSTANCE,
-      ValuesReduceRule.PROJECT_FILTER_INSTANCE,
-      ValuesReduceRule.PROJECT_INSTANCE,
-      AggregateValuesRule.INSTANCE
-  };
-
   static List<String> solrFieldNames(final RelDataType rowType) {
     return SqlValidatorUtil.uniquify(
         new AbstractList<String>() {
@@ -238,8 +224,8 @@ public RelNode convert(RelNode rel) {
       return new SolrAggregate(
           rel.getCluster(),
           traitSet,
+          agg.getHints(),
           convert(agg.getInput(), traitSet.simplify()),
-          agg.indicator,
           agg.getGroupSet(),
           agg.getGroupSets(),
           agg.getAggCallList());

solr/core/src/java/org/apache/solr/handler/sql/SolrTableScan.java

@@ -20,6 +20,7 @@
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.TableScan;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.rules.CoreRules;
 import org.apache.calcite.rel.type.RelDataType;
 
 import java.util.List;
@@ -73,9 +74,7 @@ public void register(RelOptPlanner planner) {
       planner.addRule(rule);
     }
 
-    for (RelOptRule rule : SolrRules.CONSTANT_REDUCTION_RULES) {
-      planner.addRule(rule);
-    }
+    planner.removeRule(CoreRules.FILTER_REDUCE_EXPRESSIONS); // prevent AND NOT from being reduced away, see SOLR-15461
   }
 
   public void implement(Implementor implementor) {

solr/core/src/test-files/log4j2.xml

@@ -34,6 +34,7 @@
     <Logger name="org.apache.directory" level="WARN"/>
     <Logger name="org.apache.solr.hadoop" level="INFO"/>
     <Logger name="org.eclipse.jetty" level="INFO"/>
+    <Logger name="org.apache.calcite" level="DEBUG"/>
 
     <Root level="INFO">
       <AppenderRef ref="STDERR"/>

solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java

@@ -36,6 +36,7 @@
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -474,7 +475,7 @@ public void testMixedCaseFields() throws Exception {
 
     List<Tuple> tuples = getTuples(sParams, baseUrl);
 
-    assert (tuples.size() == 8);
+    assertEquals(tuples.toString(), 8, tuples.size());
 
     Tuple tuple;
 
@@ -525,7 +526,8 @@ public void testMixedCaseFields() throws Exception {
 
     tuples = getTuples(sParams, baseUrl);
 
-    assert (tuples.size() == 2);
+
+    assertEquals(tuples.toString(), 2, tuples.size());
 
     tuple = tuples.get(0);
     assert (tuple.get("Str_s").equals("c"));
@@ -902,8 +904,7 @@ public void testSelectDistinct() throws Exception {
         "stmt", "select distinct str_s, field_i from collection1 where str_s = 'a'");
 
     tuples = getTuples(sParams, baseUrl);
-
-    assert (tuples.size() == 2);
+    Assert.assertEquals (tuples.toString(), 2, tuples.size());
 
     tuple = tuples.get(0);
     assert (tuple.get("str_s").equals("a"));
@@ -1174,7 +1175,7 @@ public void testBasicGroupingFacets() throws Exception {
     // The sort by and order by match and no limit is applied. All the Tuples should be returned in
     // this scenario.
 
-    assert (tuples.size() == 3);
+    assertEquals(tuples.toString(), 3, tuples.size());
 
     tuple = tuples.get(0);
     assert (tuple.get("str_s").equals("c"));
@@ -2054,10 +2055,10 @@ public void testDateHandling() throws Exception {
     expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex IS NOT NULL", 8);
     expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex > '2021-06-02'", 6);
     expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex <= '2021-06-01'", 1);
-    expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex BETWEEN '2021-06-03' AND '2021-06-05'", 4);
     expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex > '2021-06-04 04:00:00'", 1);
     expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex = '2021-06-04 04:00:00'", 1);
     expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex = CAST('2021-06-04 04:04:00' as TIMESTAMP)", 1);
+    expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex BETWEEN '2021-06-03' AND '2021-06-05'", 4);
   }
 
   @Test

solr/licenses/apiguardian-api-1.1.0.jar.sha1

@@ -0,0 +1 @@
+fc9dff4bb36d627bdc553de77e1f17efd790876c