From 97f457b4ede6fb34a21fad21214b4e5711f037b8 Mon Sep 17 00:00:00 2001 From: Doug Turnbull Date: Thu, 7 Jul 2016 16:47:05 -0400 Subject: [PATCH 1/6] Adds fences in stats component --- .../org/apache/solr/handler/component/StatsField.java | 11 ++++++++++- .../solr/handler/component/StatsValuesFactory.java | 6 +++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java index 4c2a2b6a5abd..7dbac57ee866 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java @@ -208,7 +208,9 @@ boolean parseParams(StatsField sf) { private final EnumSet statsInResponse = EnumSet.noneOf(Stat.class); private final List percentilesList= new ArrayList(); private final boolean isShard; - + + private double minFence = Double.MIN_VALUE; + private double maxFence = Double.MAX_VALUE; private double tdigestCompression = 100.0D; private HllOptions hllOpts; @@ -301,6 +303,9 @@ public StatsField(ResponseBuilder rb, String statsParam) { ? Collections.emptyList() : StrUtils.splitSmart(excludeStr,','); + this.minFence = localParams.getDouble("minFence", this.minFence); + this.maxFence = localParams.getDouble("maxFence", this.maxFence); + assert ( (null == this.valueSource) ^ (null == this.schemaField) ) : "exactly one of valueSource & schemaField must be null"; } @@ -512,6 +517,10 @@ public List getTagList() { return tagList; } + public double getMinFence() {return minFence; } + + public double getMaxFence() {return maxFence; } + public String toString() { return "StatsField<" + originalParam + ">"; } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index a2e4a4e78fd8..900c404263db 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -487,8 +487,12 @@ public long hash(Number v) { @Override public void accumulate(int docID) { if (values.exists(docID)) { + double minFence = statsField.getMinFence(); + double maxFence = statsField.getMaxFence(); Number value = (Number) values.objectVal(docID); - accumulate(value, 1); + if (value.doubleValue() >= minFence && value.doubleValue() <= maxFence) { + accumulate(value, 1); + } } else { missing(); } From 75ac8854285bae41014de38e073aa42d5f9f39de Mon Sep 17 00:00:00 2001 From: Doug Turnbull Date: Thu, 28 Jul 2016 16:22:48 -0400 Subject: [PATCH 2/6] Adds a test for fence testing --- .../handler/component/StatsComponentTest.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java index c0e88fc23ab5..858125c186a0 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java @@ -863,6 +863,31 @@ public void testStatsFacetMultivaluedErrorHandling() throws Exception { } + public void testStatsFencing() throws Exception { + SolrCore core = h.getCore(); + + assertU(adoc("id", "1", "foo_i", "4")); + assertU(adoc("id", "2", "foo_i", "5")); + assertU(commit()); + assertU(adoc("id", "3", "foo_i", "6")); + assertU(adoc("id", "4", "foo_i", "7")); + assertU(commit()); + + assertQ("min fence" + , req("q","*:*", "stats", "true", "stats.field", "{!minFence=5}foo_i") + ,"//lst[@name='foo_i']/double[@name='min'][.='5.0']" + ,"//lst[@name='foo_i']/double[@name='max'][.='7.0']" + ); + + + assertQ("min fence" + , req("q","*:*", "stats", "true", "stats.field", "{!maxFence=6}foo_i") + ,"//lst[@name='foo_i']/double[@name='min'][.='4.0']" + ,"//lst[@name='foo_i']/double[@name='max'][.='6.0']" + ); + + } + //SOLR-3177 public void testStatsExcludeFilterQuery() throws Exception { SolrCore core = h.getCore(); From 44763fa6a3b827a68d8c8571b01076824606334c Mon Sep 17 00:00:00 2001 From: Doug Turnbull Date: Tue, 2 Aug 2016 14:53:58 -0400 Subject: [PATCH 3/6] Change to -MAX_VALUE instead of MIN_VALUE; add "fenced" in response --- .../solr/handler/component/StatsField.java | 8 ++- .../handler/component/StatsValuesFactory.java | 68 ++++++++++++------- .../handler/component/StatsComponentTest.java | 6 ++ 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java index 7dbac57ee866..f76d2c9c6129 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java @@ -209,8 +209,9 @@ boolean parseParams(StatsField sf) { private final List percentilesList= new ArrayList(); private final boolean isShard; - private double minFence = Double.MIN_VALUE; + private double minFence = -Double.MAX_VALUE; private double maxFence = Double.MAX_VALUE; + private boolean isFenced = false; private double tdigestCompression = 100.0D; private HllOptions hllOpts; @@ -303,6 +304,9 @@ public StatsField(ResponseBuilder rb, String statsParam) { ? Collections.emptyList() : StrUtils.splitSmart(excludeStr,','); + if (localParams.get("minFence") != null || localParams.get("maxFence") != null ) { + isFenced = true; + } this.minFence = localParams.getDouble("minFence", this.minFence); this.maxFence = localParams.getDouble("maxFence", this.maxFence); @@ -521,6 +525,8 @@ public List getTagList() { public double getMaxFence() {return maxFence; } + public boolean isFenced() {return isFenced; } + public String toString() { return "StatsField<" + originalParam + ">"; } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index 900c404263db..ec1b26585699 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -89,11 +89,15 @@ public static StatsValues createStatsValues(StatsField statsField) { */ abstract class AbstractStatsValues implements StatsValues { private static final String FACETS = "facets"; - - /** Tracks all data about tthe stats we need to collect */ + + /** + * Tracks all data about tthe stats we need to collect + */ final protected StatsField statsField; - /** may be null if we are collecting stats directly from a function ValueSource */ + /** + * may be null if we are collecting stats directly from a function ValueSource + */ final protected SchemaField sf; /** * may be null if we are collecting stats directly from a function ValueSource @@ -107,11 +111,12 @@ abstract class AbstractStatsValues implements StatsValues { final protected boolean computeMin; final protected boolean computeMax; final protected boolean computeMinOrMax; - final protected boolean computeCardinality; + final protected boolean computeCardinality; + final protected boolean isFenced; - /** - * Either a function value source to collect from, or the ValueSource associated - * with a single valued field we are collecting from. Will be null until/unless + /** + * Either a function value source to collect from, or the ValueSource associated + * with a single valued field we are collecting from. Will be null until/unless * {@link #setNextReader} is called at least once */ private ValueSource valueSource; @@ -125,10 +130,11 @@ abstract class AbstractStatsValues implements StatsValues { * called at least once */ protected FunctionValues values; - + protected T max; protected T min; protected long missing; + protected long fenced; protected long count; protected long countDistinct; protected final Set distinctValues; @@ -136,27 +142,28 @@ abstract class AbstractStatsValues implements StatsValues { /** * Hash function that must be used by implementations of {@link #hash} */ - protected final HashFunction hasher; + protected final HashFunction hasher; // if null, no HLL logic can be computed; not final because of "union" optimization (see below) - private HLL hll; + private HLL hll; // facetField facetValue - protected Map> facets = new HashMap<>(); - + protected Map> facets = new HashMap<>(); + protected AbstractStatsValues(StatsField statsField) { this.statsField = statsField; this.computeCount = statsField.calculateStats(Stat.count); this.computeMissing = statsField.calculateStats(Stat.missing); - this.computeCalcDistinct = statsField.calculateStats(Stat.countDistinct) - || statsField.calculateStats(Stat.distinctValues); + this.computeCalcDistinct = statsField.calculateStats(Stat.countDistinct) + || statsField.calculateStats(Stat.distinctValues); this.computeMin = statsField.calculateStats(Stat.min); this.computeMax = statsField.calculateStats(Stat.max); + this.isFenced = statsField.isFenced(); this.computeMinOrMax = computeMin || computeMax; - + this.distinctValues = computeCalcDistinct ? new TreeSet<>() : null; this.computeCardinality = statsField.calculateStats(Stat.cardinality); - if ( computeCardinality ) { + if (computeCardinality) { hasher = statsField.getHllOptions().getHasher(); hll = statsField.getHllOptions().newHLL(); @@ -188,7 +195,7 @@ protected AbstractStatsValues(StatsField statsField) { this.ft = null; } } - + /** * {@inheritDoc} */ @@ -204,7 +211,7 @@ public void accumulate(NamedList stv) { distinctValues.addAll((Collection) stv.get("distinctValues")); countDistinct = distinctValues.size(); } - + if (computeMinOrMax) { updateMinMax((T) stv.get("min"), (T) stv.get("max")); } @@ -225,12 +232,12 @@ public void accumulate(NamedList stv) { } updateTypeSpecificStats(stv); - + NamedList f = (NamedList) stv.get(FACETS); if (f == null) { return; } - + for (int i = 0; i < f.size(); i++) { String field = f.getName(i); NamedList vals = (NamedList) f.getVal(i); @@ -250,7 +257,7 @@ public void accumulate(NamedList stv) { } } } - + /** * {@inheritDoc} */ @@ -265,7 +272,7 @@ public void accumulate(BytesRef value, int count) { accumulate(typedValue, count); } - public void accumulate(T value, int count) { + public void accumulate(T value, int count) { assert null != value : "Can't accumulate null"; if (computeCount) { @@ -281,14 +288,14 @@ public void accumulate(T value, int count) { if (computeCardinality) { if (null == hasher) { assert value instanceof Number : "pre-hashed value support only works with numeric longs"; - hll.addRaw(((Number)value).longValue()); + hll.addRaw(((Number) value).longValue()); } else { hll.addRaw(hash(value)); } } updateTypeSpecificStats(value, count); } - + /** * {@inheritDoc} */ @@ -298,6 +305,11 @@ public void missing() { missing++; } } + + public void fenced() { + fenced++; + } + /** * {@inheritDoc} @@ -490,8 +502,11 @@ public void accumulate(int docID) { double minFence = statsField.getMinFence(); double maxFence = statsField.getMaxFence(); Number value = (Number) values.objectVal(docID); - if (value.doubleValue() >= minFence && value.doubleValue() <= maxFence) { + double dVal = value.doubleValue(); + if (dVal >= minFence && dVal <= maxFence) { accumulate(value, 1); + } else if (this.isFenced) { + fenced(); } } else { missing(); @@ -576,6 +591,9 @@ protected void addTypeSpecificStats(NamedList res) { if (statsField.includeInResponse(Stat.sum)) { res.add("sum", sum); } + if (isFenced) { + res.add("fenced", fenced); + } if (statsField.includeInResponse(Stat.sumOfSquares)) { res.add("sumOfSquares", sumOfSquares); } diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java index 858125c186a0..f2364458d8d9 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java @@ -871,12 +871,15 @@ public void testStatsFencing() throws Exception { assertU(commit()); assertU(adoc("id", "3", "foo_i", "6")); assertU(adoc("id", "4", "foo_i", "7")); + assertU(adoc("id", "5")); assertU(commit()); assertQ("min fence" , req("q","*:*", "stats", "true", "stats.field", "{!minFence=5}foo_i") ,"//lst[@name='foo_i']/double[@name='min'][.='5.0']" ,"//lst[@name='foo_i']/double[@name='max'][.='7.0']" + ,"//lst[@name='foo_i']/long[@name='fenced'][.='1']" + ,"//lst[@name='foo_i']/long[@name='missing'][.='1']" ); @@ -884,10 +887,13 @@ public void testStatsFencing() throws Exception { , req("q","*:*", "stats", "true", "stats.field", "{!maxFence=6}foo_i") ,"//lst[@name='foo_i']/double[@name='min'][.='4.0']" ,"//lst[@name='foo_i']/double[@name='max'][.='6.0']" + ,"//lst[@name='foo_i']/long[@name='fenced'][.='1']" + ,"//lst[@name='foo_i']/long[@name='missing'][.='1']" ); } + //SOLR-3177 public void testStatsExcludeFilterQuery() throws Exception { SolrCore core = h.getCore(); From de817bdad4fcb0747f6565ff74593ab04d2ac574 Mon Sep 17 00:00:00 2001 From: Doug Turnbull Date: Sat, 6 Aug 2016 21:45:19 -0400 Subject: [PATCH 4/6] Refactors "fencing" to ceil/floor bounding --- .../solr/handler/component/StatsField.java | 20 +++---- .../solr/handler/component/StatsValues.java | 7 +++ .../handler/component/StatsValuesFactory.java | 59 ++++++++++++++----- .../handler/component/StatsComponentTest.java | 44 ++++++++++++-- 4 files changed, 97 insertions(+), 33 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java index f76d2c9c6129..0a9bc6cb4c02 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java @@ -209,9 +209,8 @@ boolean parseParams(StatsField sf) { private final List percentilesList= new ArrayList(); private final boolean isShard; - private double minFence = -Double.MAX_VALUE; - private double maxFence = Double.MAX_VALUE; - private boolean isFenced = false; + private String floor; + private String ceil; private double tdigestCompression = 100.0D; private HllOptions hllOpts; @@ -304,11 +303,10 @@ public StatsField(ResponseBuilder rb, String statsParam) { ? Collections.emptyList() : StrUtils.splitSmart(excludeStr,','); - if (localParams.get("minFence") != null || localParams.get("maxFence") != null ) { - isFenced = true; - } - this.minFence = localParams.getDouble("minFence", this.minFence); - this.maxFence = localParams.getDouble("maxFence", this.maxFence); + // schema-type specific, so these are pulled out as strings and + // dealt with by the stats values + this.floor = localParams.get("floor"); + this.ceil = localParams.get("ceil"); assert ( (null == this.valueSource) ^ (null == this.schemaField) ) : "exactly one of valueSource & schemaField must be null"; @@ -521,11 +519,9 @@ public List getTagList() { return tagList; } - public double getMinFence() {return minFence; } - - public double getMaxFence() {return maxFence; } + public String getFloor() {return floor;} - public boolean isFenced() {return isFenced; } + public String getCeil() {return ceil; } public String toString() { return "StatsField<" + originalParam + ">"; diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java index 8c85fda7a71b..f32c30d276a2 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java @@ -54,6 +54,13 @@ public interface StatsValues { */ void missing(); + /** + * Updates the statistics for a document that was excluded as + * it was out of bounds + */ + + void outOfBounds(); + /** * Updates the statistics when multiple documents are missing a value * diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index ec1b26585699..472efa38628e 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -112,7 +112,16 @@ abstract class AbstractStatsValues implements StatsValues { final protected boolean computeMax; final protected boolean computeMinOrMax; final protected boolean computeCardinality; - final protected boolean isFenced; + + interface Boundable { + boolean isWithinBounds(T value); + } + + protected Boundable bounds = new Boundable() { + public boolean isWithinBounds(T value) { + return true; + } + }; /** * Either a function value source to collect from, or the ValueSource associated @@ -134,7 +143,7 @@ abstract class AbstractStatsValues implements StatsValues { protected T max; protected T min; protected long missing; - protected long fenced; + protected long outOfBounds; protected long count; protected long countDistinct; protected final Set distinctValues; @@ -157,7 +166,6 @@ protected AbstractStatsValues(StatsField statsField) { || statsField.calculateStats(Stat.distinctValues); this.computeMin = statsField.calculateStats(Stat.min); this.computeMax = statsField.calculateStats(Stat.max); - this.isFenced = statsField.isFenced(); this.computeMinOrMax = computeMin || computeMax; this.distinctValues = computeCalcDistinct ? new TreeSet<>() : null; @@ -275,6 +283,11 @@ public void accumulate(BytesRef value, int count) { public void accumulate(T value, int count) { assert null != value : "Can't accumulate null"; + if (!bounds.isWithinBounds(value)) { + outOfBounds(); + return; + } + if (computeCount) { this.count += count; } @@ -306,8 +319,8 @@ public void missing() { } } - public void fenced() { - fenced++; + public void outOfBounds() { + outOfBounds++; } @@ -456,7 +469,7 @@ class NumericStatsValues extends AbstractStatsValues { double minD; // perf optimization, only valid if (null != this.min) double maxD; // perf optimization, only valid if (null != this.max) - + final protected boolean computeSum; final protected boolean computeSumOfSquares; final protected boolean computePercentiles; @@ -472,6 +485,25 @@ public NumericStatsValues(StatsField statsField) { tdigest = new AVLTreeDigest(statsField.getTdigestCompression()); } + + if (statsField.getCeil() != null || statsField.getFloor() != null) { + double floor = statsField.getFloor() != null ? Double.parseDouble(statsField.getFloor()) : -Double.MAX_VALUE; + double ceil = statsField.getCeil() != null ? Double.parseDouble(statsField.getCeil()) : +Double.MAX_VALUE; + + bounds = new Boundable() { + @Override + public boolean isWithinBounds(Number value) { + // really we should think harder about when things are longs, but in accumulate + // stats component only cares about double, so we do here as well + if (value.doubleValue() >= floor && value.doubleValue() <= ceil) { + return true; + } + return false; + } + }; + } + + } @Override @@ -499,15 +531,9 @@ public long hash(Number v) { @Override public void accumulate(int docID) { if (values.exists(docID)) { - double minFence = statsField.getMinFence(); - double maxFence = statsField.getMaxFence(); Number value = (Number) values.objectVal(docID); double dVal = value.doubleValue(); - if (dVal >= minFence && dVal <= maxFence) { - accumulate(value, 1); - } else if (this.isFenced) { - fenced(); - } + accumulate(value, 1); } else { missing(); } @@ -591,8 +617,8 @@ protected void addTypeSpecificStats(NamedList res) { if (statsField.includeInResponse(Stat.sum)) { res.add("sum", sum); } - if (isFenced) { - res.add("fenced", fenced); + if (outOfBounds > 0) { + res.add("outOfBounds", outOfBounds); } if (statsField.includeInResponse(Stat.sumOfSquares)) { res.add("sumOfSquares", sumOfSquares); @@ -726,7 +752,8 @@ class DateStatsValues extends AbstractStatsValues { private double sum = 0.0; double sumOfSquares = 0; - + + final protected boolean computeSum; final protected boolean computeSumOfSquares; diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java index f2364458d8d9..0cc1c09293a9 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java @@ -875,26 +875,60 @@ public void testStatsFencing() throws Exception { assertU(commit()); assertQ("min fence" - , req("q","*:*", "stats", "true", "stats.field", "{!minFence=5}foo_i") + , req("q","*:*", "stats", "true", "stats.field", "{!floor=5}foo_i") ,"//lst[@name='foo_i']/double[@name='min'][.='5.0']" ,"//lst[@name='foo_i']/double[@name='max'][.='7.0']" - ,"//lst[@name='foo_i']/long[@name='fenced'][.='1']" + ,"//lst[@name='foo_i']/long[@name='outOfBounds'][.='1']" ,"//lst[@name='foo_i']/long[@name='missing'][.='1']" ); assertQ("min fence" - , req("q","*:*", "stats", "true", "stats.field", "{!maxFence=6}foo_i") + , req("q","*:*", "stats", "true", "stats.field", "{!ceil=6}foo_i") ,"//lst[@name='foo_i']/double[@name='min'][.='4.0']" ,"//lst[@name='foo_i']/double[@name='max'][.='6.0']" - ,"//lst[@name='foo_i']/long[@name='fenced'][.='1']" + ,"//lst[@name='foo_i']/long[@name='outOfBounds'][.='1']" ,"//lst[@name='foo_i']/long[@name='missing'][.='1']" ); + } + + public void testStatsDateFencing() throws Exception { + SolrCore core = h.getCore(); + + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + + String date1 = dateFormat.format(new Date(123456789)) + "Z"; + String date2 = dateFormat.format(new Date(987654321)) + "Z"; + + assertU(adoc("id", "1", "active_dt", date1)); + assertU(adoc("id", "2", "active_dt", date2)); + assertU(adoc("id", "3")); + assertU(commit()); + + Map args = new HashMap<>(); + args.put(CommonParams.Q, "*:*"); + args.put(StatsParams.STATS, "true"); + args.put(StatsParams.STATS_FIELD, "{!maxFence=" + dateFormat.format(new Date(987654320)) + "Z}active_dt"); + args.put("f.active_dt.stats.calcdistinct","true"); + args.put("indent", "true"); + SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args)); + + assertQ("test date statistics values", req, + "//long[@name='count'][.='1']", + "//long[@name='missing'][.='1']", + "//long[@name='outOfBounds'][.='1']", + "//date[@name='min'][.='1970-01-02T10:17:36Z']", + "//date[@name='max'][.='1970-01-02T10:17:36Z']", + "//long[@name='countDistinct'][.='2']", + "count(//arr[@name='distinctValues']/date)=2"); + + } - //SOLR-3177 + //SOLR-3177 public void testStatsExcludeFilterQuery() throws Exception { SolrCore core = h.getCore(); assertU(adoc("id", "1")); From 02ae7c6ba3f0e22c51d08c3ca3dfe5d469301bc5 Mon Sep 17 00:00:00 2001 From: Doug Turnbull Date: Sat, 6 Aug 2016 22:07:59 -0400 Subject: [PATCH 5/6] Adds test for date bounds --- .../handler/component/StatsValuesFactory.java | 28 +++++++++++++++++-- .../handler/component/StatsComponentTest.java | 7 ++--- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index 472efa38628e..73dbb6222b55 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -33,6 +33,7 @@ import com.tdunning.math.stats.AVLTreeDigest; import com.google.common.hash.HashFunction; +import org.apache.solr.util.DateMathParser; import org.apache.solr.util.hll.HLL; import org.apache.solr.util.hll.HLLType; @@ -372,6 +373,9 @@ public NamedList getStatsValues() { res.add("cardinality", hll.cardinality()); } } + if (outOfBounds > 0) { + res.add("outOfBounds", outOfBounds); + } addTypeSpecificStats(res); @@ -617,9 +621,6 @@ protected void addTypeSpecificStats(NamedList res) { if (statsField.includeInResponse(Stat.sum)) { res.add("sum", sum); } - if (outOfBounds > 0) { - res.add("outOfBounds", outOfBounds); - } if (statsField.includeInResponse(Stat.sumOfSquares)) { res.add("sumOfSquares", sumOfSquares); } @@ -761,6 +762,27 @@ public DateStatsValues(StatsField statsField) { super(statsField); this.computeSum = statsField.calculateStats(Stat.sum); this.computeSumOfSquares = statsField.calculateStats(Stat.sumOfSquares); + + if (statsField.getCeil() != null || statsField.getFloor() != null) { + Date floor = statsField.getFloor() != null ? DateMathParser.parseMath(null, statsField.getFloor()) : null; + Date ceil = statsField.getCeil() != null ? DateMathParser.parseMath(null, statsField.getCeil()) : null; + + bounds = new Boundable() { + @Override + public boolean isWithinBounds(Date value) { + // really we should think harder about when things are longs, but in accumulate + // stats component only cares about double, so we do here as well + if (floor != null && (value.compareTo(floor) <= 0)) { + return false; + } + + if (ceil != null && value.compareTo(ceil) >= 0) { + return false; + } + return true; + } + }; + } } @Override diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java index 0cc1c09293a9..049de7a5b1e2 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java @@ -910,8 +910,7 @@ public void testStatsDateFencing() throws Exception { Map args = new HashMap<>(); args.put(CommonParams.Q, "*:*"); args.put(StatsParams.STATS, "true"); - args.put(StatsParams.STATS_FIELD, "{!maxFence=" + dateFormat.format(new Date(987654320)) + "Z}active_dt"); - args.put("f.active_dt.stats.calcdistinct","true"); + args.put(StatsParams.STATS_FIELD, "{!ceil=" + dateFormat.format(new Date(987654320)) + "Z}active_dt"); args.put("indent", "true"); SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args)); @@ -920,9 +919,7 @@ public void testStatsDateFencing() throws Exception { "//long[@name='missing'][.='1']", "//long[@name='outOfBounds'][.='1']", "//date[@name='min'][.='1970-01-02T10:17:36Z']", - "//date[@name='max'][.='1970-01-02T10:17:36Z']", - "//long[@name='countDistinct'][.='2']", - "count(//arr[@name='distinctValues']/date)=2"); + "//date[@name='max'][.='1970-01-02T10:17:36Z']"); } From fb6142c998207df022a9cc16fda88fc9aeb8fefb Mon Sep 17 00:00:00 2001 From: Doug Turnbull Date: Mon, 8 Aug 2016 15:42:00 -0400 Subject: [PATCH 6/6] Changes boundables to lambdas --- .../handler/component/StatsValuesFactory.java | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index 73dbb6222b55..17132d6ea164 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -118,12 +118,7 @@ interface Boundable { boolean isWithinBounds(T value); } - protected Boundable bounds = new Boundable() { - public boolean isWithinBounds(T value) { - return true; - } - }; - + protected Boundable bounds = (T) -> true; /** * Either a function value source to collect from, or the ValueSource associated * with a single valued field we are collecting from. Will be null until/unless @@ -494,16 +489,13 @@ public NumericStatsValues(StatsField statsField) { double floor = statsField.getFloor() != null ? Double.parseDouble(statsField.getFloor()) : -Double.MAX_VALUE; double ceil = statsField.getCeil() != null ? Double.parseDouble(statsField.getCeil()) : +Double.MAX_VALUE; - bounds = new Boundable() { - @Override - public boolean isWithinBounds(Number value) { + bounds = (Number value) -> { // really we should think harder about when things are longs, but in accumulate // stats component only cares about double, so we do here as well if (value.doubleValue() >= floor && value.doubleValue() <= ceil) { return true; } return false; - } }; } @@ -767,20 +759,17 @@ public DateStatsValues(StatsField statsField) { Date floor = statsField.getFloor() != null ? DateMathParser.parseMath(null, statsField.getFloor()) : null; Date ceil = statsField.getCeil() != null ? DateMathParser.parseMath(null, statsField.getCeil()) : null; - bounds = new Boundable() { - @Override - public boolean isWithinBounds(Date value) { - // really we should think harder about when things are longs, but in accumulate - // stats component only cares about double, so we do here as well - if (floor != null && (value.compareTo(floor) <= 0)) { - return false; - } + bounds = (Date value) -> { + // really we should think harder about when things are longs, but in accumulate + // stats component only cares about double, so we do here as well + if (floor != null && (value.compareTo(floor) <= 0)) { + return false; + } - if (ceil != null && value.compareTo(ceil) >= 0) { - return false; - } - return true; + if (ceil != null && value.compareTo(ceil) >= 0) { + return false; } + return true; }; } }