From f2f1ecd907f292536d5713dfcf6f4afca433f0e2 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Feb 2025 11:10:16 +0100 Subject: [PATCH 1/8] Add histogram facet capabilities. This is inspired from a paper by Tencent where the authors describe how they speed up so-called "histogram queries" by sorting the index by timestamp translating ranges of values corresponding to each histogram bucket to ranges of doc IDs. This way, at collection time, they no longer need to look up values and can compute the histogram purely by looking at collected doc IDs. YU, Muzhi, LIN, Zhaoxiang, SUN, Jinan, et al. TencentCLS: the cloud log service with high query performances. Proceedings of the VLDB Endowment, 2022, vol. 15, no 12, p. 3472-3482. Instead of binary-searching the doc ID space to translate histogram buckets into ranges of doc IDs, the new collector manager uses recently introduced support for sparse indexing. When playing with the geonames dataset, computing a histogram of the elevation field runs ~2-3x faster with this optimization than with the naive implementation. --- .../facet/histogram/HistogramCollector.java | 252 ++++++++++++++++++ .../histogram/HistogramCollectorManager.java | 72 +++++ .../TestHistogramCollectorManager.java | 128 +++++++++ 3 files changed, 452 insertions(+) create mode 100644 lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java create mode 100644 lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java create mode 100644 lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java diff --git a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java b/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java new file mode 100644 index 000000000000..18b9143d16a3 --- /dev/null +++ b/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.facet.histogram; + +import java.io.IOException; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.internal.hppc.LongIntHashMap; +import org.apache.lucene.search.CollectionTerminatedException; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; + +final class HistogramCollector implements Collector { + + private final String field; + private final long interval; + private final LongIntHashMap counts; + + HistogramCollector(String field, long interval) { + this.field = field; + this.interval = interval; + this.counts = new LongIntHashMap(); + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + FieldInfo fi = context.reader().getFieldInfos().fieldInfo(field); + if (fi == null) { + // The segment has no values, nothing to do. + throw new CollectionTerminatedException(); + } + if (fi.getDocValuesType() != DocValuesType.NUMERIC + && fi.getDocValuesType() != DocValuesType.SORTED_NUMERIC) { + throw new IllegalStateException( + "Expected numeric field, but got doc-value type: " + fi.getDocValuesType()); + } + SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); + NumericDocValues singleton = DocValues.unwrapSingleton(values); + if (singleton == null) { + return new HistogramNaiveLeafCollector(values, interval, counts); + } else { + DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); + if (skipper != null) { + long leafMinQuotient = Math.floorDiv(skipper.minValue(), interval); + long leafMaxQuotient = Math.floorDiv(skipper.maxValue(), interval); + if (leafMaxQuotient - leafMinQuotient <= 1024) { + // Only use the optimized implementation if there is a small number of unique quotients, + // so that we can count them using a dense array instead of a hash table. + return new HistogramLeafCollector(singleton, skipper, interval, counts); + } + } + return new HistogramNaiveSingleValuedLeafCollector(singleton, interval, counts); + } + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + LongIntHashMap getCounts() { + return counts; + } + + /** + * Naive implementation of a histogram {@link LeafCollector}, which iterates all maches and looks + * up the value to determine the corresponding bucket. + */ + private static class HistogramNaiveLeafCollector implements LeafCollector { + + private final SortedNumericDocValues values; + private final long interval; + private final LongIntHashMap counts; + + HistogramNaiveLeafCollector( + SortedNumericDocValues values, long interval, LongIntHashMap counts) { + this.values = values; + this.interval = interval; + this.counts = counts; + } + + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + if (values.advanceExact(doc)) { + int valueCount = values.docValueCount(); + long prevQuotient = Long.MIN_VALUE; + for (int i = 0; i < valueCount; ++i) { + final long value = values.nextValue(); + final long quotient = Math.floorDiv(value, interval); + // We must not double-count values that divide to the same quotient since this returns doc + // counts as opposed to value counts. + if (quotient != prevQuotient) { + counts.addTo(quotient, 1); + prevQuotient = quotient; + } + } + } + } + } + + /** + * Naive implementation of a histogram {@link LeafCollector}, which iterates all maches and looks + * up the value to determine the corresponding bucket. + */ + private static class HistogramNaiveSingleValuedLeafCollector implements LeafCollector { + + private final NumericDocValues values; + private final long interval; + private final LongIntHashMap counts; + + HistogramNaiveSingleValuedLeafCollector( + NumericDocValues values, long interval, LongIntHashMap counts) { + this.values = values; + this.interval = interval; + this.counts = counts; + } + + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + if (values.advanceExact(doc)) { + final long value = values.longValue(); + final long quotient = Math.floorDiv(value, interval); + counts.addTo(quotient, 1); + } + } + } + + /** + * Optimized histogram {@link LeafCollector}, that takes advantage of the doc-values index to + * speed up collection. + */ + private static class HistogramLeafCollector implements LeafCollector { + + private final NumericDocValues values; + private final DocValuesSkipper skipper; + private final long interval; + private final int[] counts; + private final long leafMinQuotient; + private final LongIntHashMap collectorCounts; + + /** + * Max doc ID (inclusive) up to which all docs may map to values that have the same quotient. + */ + private int upToInclusive = -1; + + /** Whether all docs up to {@link #upToInclusive} map to values that have the same quotient. */ + private boolean upToSameQuotient; + + /** Index in {@link #counts} for docs up to {@link #upToInclusive}. */ + private int upToQuotientIndex; + + HistogramLeafCollector( + NumericDocValues values, + DocValuesSkipper skipper, + long interval, + LongIntHashMap collectorCounts) { + this.values = values; + this.skipper = skipper; + this.interval = interval; + this.collectorCounts = collectorCounts; + + leafMinQuotient = Math.floorDiv(skipper.minValue(), interval); + long leafMaxQuotient = Math.floorDiv(skipper.maxValue(), interval); + counts = new int[Math.toIntExact(leafMaxQuotient - leafMinQuotient + 1)]; + } + + @Override + public void setScorer(Scorable scorer) throws IOException {} + + private void advanceSkipper(int doc) throws IOException { + if (doc > skipper.maxDocID(0)) { + skipper.advance(doc); + } + upToSameQuotient = false; + + if (skipper.minDocID(0) > doc) { + // Corner case which happens if `doc` doesn't have a value and is between two intervals of + // the doc-value skip index. + upToInclusive = skipper.minDocID(0) - 1; + return; + } + + upToInclusive = skipper.maxDocID(0); + + // Now find the highest level where all docs have the same quotient. + for (int level = 0; level < skipper.numLevels(); ++level) { + int totalDocsAtLevel = skipper.maxDocID(level) - skipper.minDocID(level) + 1; + long minQuotient = Math.floorDiv(skipper.minValue(level), interval); + long maxQuotient = Math.floorDiv(skipper.maxValue(level), interval); + + if (skipper.docCount(level) == totalDocsAtLevel && minQuotient == maxQuotient) { + // All docs at this level have a value, and all values map to the same quotient. + upToInclusive = skipper.maxDocID(level); + upToSameQuotient = true; + upToQuotientIndex = (int) (minQuotient - this.leafMinQuotient); + } else { + break; + } + } + } + + @Override + public void collect(int doc) throws IOException { + if (doc > upToInclusive) { + advanceSkipper(doc); + } + + if (upToSameQuotient) { + counts[upToQuotientIndex]++; + } else if (values.advanceExact(doc)) { + final long value = values.longValue(); + final long quotient = Math.floorDiv(value, interval); + counts[(int) (quotient - leafMinQuotient)]++; + } + } + + @Override + public void finish() throws IOException { + // Put counts that we computed in the int[] back into the hash map. + for (int i = 0; i < counts.length; ++i) { + collectorCounts.addTo(leafMinQuotient + i, counts[i]); + } + } + } +} diff --git a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java b/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java new file mode 100644 index 000000000000..c4dfc7fe4c4d --- /dev/null +++ b/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.facet.histogram; + +import java.io.IOException; +import java.util.Collection; +import java.util.Objects; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.internal.hppc.LongIntHashMap; +import org.apache.lucene.internal.hppc.LongIntHashMap.LongIntCursor; +import org.apache.lucene.search.CollectorManager; + +/** + * {@link CollectorManager} that computes a histogram of the distribution of the values of a field. + * + *

The returned {@link LongIntHashMap} maps quotients to the number of documents whose value + * returns this number when divided by the given {@code interval}. + * + *

This implementation is optimized for the case when {@code field} is part of the index sort and + * has a {@link FieldType#setDocValuesSkipIndexType skip index}. + * + *

Note: this collector is inspired from "YU, Muzhi, LIN, Zhaoxiang, SUN, Jinan, et al. + * TencentCLS: the cloud log service with high query performances. Proceedings of the VLDB + * Endowment, 2022, vol. 15, no 12, p. 3472-3482.", where the authors describe how they run + * "histogram queries" by sorting the index by timestamp and pre-computing ranges of doc IDs for + * every possible bucket. + */ +public final class HistogramCollectorManager + implements CollectorManager { + + private final String field; + private final long interval; + + /** Sole constructor. */ + public HistogramCollectorManager(String field, long interval) { + this.field = Objects.requireNonNull(field); + this.interval = interval; + if (interval < 2) { + throw new IllegalArgumentException("interval must be at least 2, got: " + interval); + } + } + + @Override + public HistogramCollector newCollector() throws IOException { + return new HistogramCollector(field, interval); + } + + @Override + public LongIntHashMap reduce(Collection collectors) throws IOException { + LongIntHashMap reduced = new LongIntHashMap(); + for (HistogramCollector collector : collectors) { + for (LongIntCursor cursor : collector.getCounts()) { + reduced.addTo(cursor.key, cursor.value); + } + } + return reduced; + } +} diff --git a/lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java b/lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java new file mode 100644 index 000000000000..aa26b117b26d --- /dev/null +++ b/lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.facet.histogram; + +import java.io.IOException; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.internal.hppc.LongIntHashMap; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; + +public class TestHistogramCollectorManager extends LuceneTestCase { + + public void testSingleValuedNoSkipIndex() throws IOException { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()); + Document doc = new Document(); + doc.add(new NumericDocValuesField("f", 3)); + w.addDocument(doc); + doc = new Document(); + doc.add(new NumericDocValuesField("f", 4)); + w.addDocument(doc); + doc = new Document(); + doc.add(new NumericDocValuesField("f", 6)); + w.addDocument(doc); + DirectoryReader reader = DirectoryReader.open(w); + w.close(); + IndexSearcher searcher = newSearcher(reader); + LongIntHashMap actualCounts = + searcher.search(new MatchAllDocsQuery(), new HistogramCollectorManager("f", 4)); + LongIntHashMap expectedCounts = new LongIntHashMap(); + expectedCounts.put(0, 1); + expectedCounts.put(1, 2); + assertEquals(expectedCounts, actualCounts); + reader.close(); + dir.close(); + } + + public void testMultiValuedNoSkipIndex() throws IOException { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()); + Document doc = new Document(); + doc.add(new SortedNumericDocValuesField("f", 3)); + doc.add(new SortedNumericDocValuesField("f", 8)); + w.addDocument(doc); + doc = new Document(); + doc.add(new SortedNumericDocValuesField("f", 4)); + doc.add(new SortedNumericDocValuesField("f", 6)); + doc.add(new SortedNumericDocValuesField("f", 8)); + w.addDocument(doc); + DirectoryReader reader = DirectoryReader.open(w); + w.close(); + IndexSearcher searcher = newSearcher(reader); + LongIntHashMap actualCounts = + searcher.search(new MatchAllDocsQuery(), new HistogramCollectorManager("f", 4)); + LongIntHashMap expectedCounts = new LongIntHashMap(); + expectedCounts.put(0, 1); + expectedCounts.put(1, 1); + expectedCounts.put(2, 2); + assertEquals(expectedCounts, actualCounts); + reader.close(); + dir.close(); + } + + public void testSkipIndex() throws IOException { + doTestSkipIndex(newIndexWriterConfig()); + } + + public void testSkipIndexWithSort() throws IOException { + doTestSkipIndex( + newIndexWriterConfig().setIndexSort(new Sort(new SortField("f", SortField.Type.LONG)))); + } + + public void testSkipIndexWithSortAndLowInterval() throws IOException { + doTestSkipIndex( + newIndexWriterConfig() + .setIndexSort(new Sort(new SortField("f", SortField.Type.LONG))) + .setCodec(TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(3)))); + } + + private void doTestSkipIndex(IndexWriterConfig cfg) throws IOException { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, cfg); + long[] values = new long[] {3, 6, 0, 4, 6, 12, 8, 8, 7, 8, 0, 4, 3, 6, 11}; + for (long value : values) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("f", value)); + w.addDocument(doc); + } + + DirectoryReader reader = DirectoryReader.open(w); + w.close(); + IndexSearcher searcher = newSearcher(reader); + LongIntHashMap actualCounts = + searcher.search(new MatchAllDocsQuery(), new HistogramCollectorManager("f", 4)); + LongIntHashMap expectedCounts = new LongIntHashMap(); + for (long value : values) { + expectedCounts.addTo(Math.floorDiv(value, 4), 1); + } + assertEquals(expectedCounts, actualCounts); + reader.close(); + dir.close(); + } +} From 520da4a197bcfe1ac260b7135a0323a045693784 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Feb 2025 13:13:17 +0100 Subject: [PATCH 2/8] Add package javadocs. --- .../lucene/facet/histogram/package-info.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lucene/facet/src/java/org/apache/lucene/facet/histogram/package-info.java diff --git a/lucene/facet/src/java/org/apache/lucene/facet/histogram/package-info.java b/lucene/facet/src/java/org/apache/lucene/facet/histogram/package-info.java new file mode 100644 index 000000000000..f2dfbc1dc74e --- /dev/null +++ b/lucene/facet/src/java/org/apache/lucene/facet/histogram/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Provides histotgram faceting capabilities. */ +package org.apache.lucene.facet.histogram; From 9aef76b56fee33e8f8113f295a672061c6f77a7c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Feb 2025 18:30:44 +0100 Subject: [PATCH 3/8] Add missing module export --- lucene/facet/src/java/module-info.java | 1 + 1 file changed, 1 insertion(+) diff --git a/lucene/facet/src/java/module-info.java b/lucene/facet/src/java/module-info.java index 2aa1e3e494e5..0dfaea136d61 100644 --- a/lucene/facet/src/java/module-info.java +++ b/lucene/facet/src/java/module-info.java @@ -20,6 +20,7 @@ requires org.apache.lucene.core; exports org.apache.lucene.facet; + exports org.apache.lucene.facet.histogram; exports org.apache.lucene.facet.range; exports org.apache.lucene.facet.sortedset; exports org.apache.lucene.facet.taxonomy; From 21498ff200b42cfdde5239d7821ab45a63a2a814 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 7 Feb 2025 22:59:43 +0100 Subject: [PATCH 4/8] iter --- .../facet/histogram/HistogramCollector.java | 34 +++++++++++++++---- .../histogram/HistogramCollectorManager.java | 27 +++++++++++++-- .../TestHistogramCollectorManager.java | 15 ++++++++ 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java b/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java index 18b9143d16a3..f443db6b1f84 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java @@ -35,11 +35,13 @@ final class HistogramCollector implements Collector { private final String field; private final long interval; + private final int maxBuckets; private final LongIntHashMap counts; - HistogramCollector(String field, long interval) { + HistogramCollector(String field, long interval, int maxBuckets) { this.field = field; this.interval = interval; + this.maxBuckets = maxBuckets; this.counts = new LongIntHashMap(); } @@ -58,7 +60,7 @@ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOExcept SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); NumericDocValues singleton = DocValues.unwrapSingleton(values); if (singleton == null) { - return new HistogramNaiveLeafCollector(values, interval, counts); + return new HistogramNaiveLeafCollector(values, interval, maxBuckets, counts); } else { DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); if (skipper != null) { @@ -67,10 +69,10 @@ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOExcept if (leafMaxQuotient - leafMinQuotient <= 1024) { // Only use the optimized implementation if there is a small number of unique quotients, // so that we can count them using a dense array instead of a hash table. - return new HistogramLeafCollector(singleton, skipper, interval, counts); + return new HistogramLeafCollector(singleton, skipper, interval, maxBuckets, counts); } } - return new HistogramNaiveSingleValuedLeafCollector(singleton, interval, counts); + return new HistogramNaiveSingleValuedLeafCollector(singleton, interval, maxBuckets, counts); } } @@ -91,12 +93,14 @@ private static class HistogramNaiveLeafCollector implements LeafCollector { private final SortedNumericDocValues values; private final long interval; + private final int maxBuckets; private final LongIntHashMap counts; HistogramNaiveLeafCollector( - SortedNumericDocValues values, long interval, LongIntHashMap counts) { + SortedNumericDocValues values, long interval, int maxBuckets, LongIntHashMap counts) { this.values = values; this.interval = interval; + this.maxBuckets = maxBuckets; this.counts = counts; } @@ -115,6 +119,7 @@ public void collect(int doc) throws IOException { // counts as opposed to value counts. if (quotient != prevQuotient) { counts.addTo(quotient, 1); + checkMaxBuckets(counts.size(), maxBuckets); prevQuotient = quotient; } } @@ -130,12 +135,14 @@ private static class HistogramNaiveSingleValuedLeafCollector implements LeafColl private final NumericDocValues values; private final long interval; + private final int maxBuckets; private final LongIntHashMap counts; HistogramNaiveSingleValuedLeafCollector( - NumericDocValues values, long interval, LongIntHashMap counts) { + NumericDocValues values, long interval, int maxBuckets, LongIntHashMap counts) { this.values = values; this.interval = interval; + this.maxBuckets = maxBuckets; this.counts = counts; } @@ -148,6 +155,7 @@ public void collect(int doc) throws IOException { final long value = values.longValue(); final long quotient = Math.floorDiv(value, interval); counts.addTo(quotient, 1); + checkMaxBuckets(counts.size(), maxBuckets); } } } @@ -161,6 +169,7 @@ private static class HistogramLeafCollector implements LeafCollector { private final NumericDocValues values; private final DocValuesSkipper skipper; private final long interval; + private final int maxBuckets; private final int[] counts; private final long leafMinQuotient; private final LongIntHashMap collectorCounts; @@ -180,10 +189,12 @@ private static class HistogramLeafCollector implements LeafCollector { NumericDocValues values, DocValuesSkipper skipper, long interval, + int maxBuckets, LongIntHashMap collectorCounts) { this.values = values; this.skipper = skipper; this.interval = interval; + this.maxBuckets = maxBuckets; this.collectorCounts = collectorCounts; leafMinQuotient = Math.floorDiv(skipper.minValue(), interval); @@ -247,6 +258,17 @@ public void finish() throws IOException { for (int i = 0; i < counts.length; ++i) { collectorCounts.addTo(leafMinQuotient + i, counts[i]); } + checkMaxBuckets(collectorCounts.size(), maxBuckets); + } + } + + private static void checkMaxBuckets(int size, int maxBuckets) { + if (size > maxBuckets) { + throw new IllegalStateException( + "Collected " + + size + + " buckets, which is more than the configured max number of buckets: " + + maxBuckets); } } } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java b/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java index c4dfc7fe4c4d..e14de49d3b4c 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java @@ -42,21 +42,42 @@ public final class HistogramCollectorManager implements CollectorManager { + private static final int DEFAULT_MAX_BUCKETS = 1024; + private final String field; private final long interval; + private final int maxBuckets; - /** Sole constructor. */ + /** + * Compute a histogram of the distribution of the values of the given {@code field} according to + * the given {@code interval}. This configures a maximum number of buckets equal to the default of + * 1024. + */ public HistogramCollectorManager(String field, long interval) { + this(field, interval, DEFAULT_MAX_BUCKETS); + } + + /** + * Expert constructor. + * + * @param maxBuckets Max allowed number of buckets. Note that this is checked at runtime and on a + * best-effort basis. + */ + public HistogramCollectorManager(String field, long interval, int maxBuckets) { this.field = Objects.requireNonNull(field); - this.interval = interval; if (interval < 2) { throw new IllegalArgumentException("interval must be at least 2, got: " + interval); } + this.interval = interval; + if (maxBuckets < 1) { + throw new IllegalArgumentException("maxBuckets must be at least 1, got: " + maxBuckets); + } + this.maxBuckets = maxBuckets; } @Override public HistogramCollector newCollector() throws IOException { - return new HistogramCollector(field, interval); + return new HistogramCollector(field, interval, maxBuckets); } @Override diff --git a/lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java b/lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java index aa26b117b26d..d121825a31ac 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java @@ -56,6 +56,11 @@ public void testSingleValuedNoSkipIndex() throws IOException { expectedCounts.put(0, 1); expectedCounts.put(1, 2); assertEquals(expectedCounts, actualCounts); + + expectThrows( + IllegalStateException.class, + () -> searcher.search(new MatchAllDocsQuery(), new HistogramCollectorManager("f", 4, 1))); + reader.close(); dir.close(); } @@ -82,6 +87,11 @@ public void testMultiValuedNoSkipIndex() throws IOException { expectedCounts.put(1, 1); expectedCounts.put(2, 2); assertEquals(expectedCounts, actualCounts); + + expectThrows( + IllegalStateException.class, + () -> searcher.search(new MatchAllDocsQuery(), new HistogramCollectorManager("f", 4, 1))); + reader.close(); dir.close(); } @@ -122,6 +132,11 @@ private void doTestSkipIndex(IndexWriterConfig cfg) throws IOException { expectedCounts.addTo(Math.floorDiv(value, 4), 1); } assertEquals(expectedCounts, actualCounts); + + expectThrows( + IllegalStateException.class, + () -> searcher.search(new MatchAllDocsQuery(), new HistogramCollectorManager("f", 4, 1))); + reader.close(); dir.close(); } From d265bafaa1301b25babed6ab6ba8fe976631039c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Feb 2025 11:41:25 +0100 Subject: [PATCH 5/8] Move to sandbox, more docs. --- lucene/facet/src/java/module-info.java | 1 - lucene/sandbox/src/java/module-info.java | 1 + .../plain/histograms}/HistogramCollector.java | 2 +- .../HistogramCollectorManager.java | 9 ++++--- .../facet/plain/histograms}/package-info.java | 2 +- .../sandbox/facet/plain/package-info.java | 24 +++++++++++++++++++ .../TestHistogramCollectorManager.java | 2 +- 7 files changed, 34 insertions(+), 7 deletions(-) rename lucene/{facet/src/java/org/apache/lucene/facet/histogram => sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms}/HistogramCollector.java (99%) rename lucene/{facet/src/java/org/apache/lucene/facet/histogram => sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms}/HistogramCollectorManager.java (88%) rename lucene/{facet/src/java/org/apache/lucene/facet/histogram => sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms}/package-info.java (93%) create mode 100644 lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/package-info.java rename lucene/{facet/src/test/org/apache/lucene/facet/histogram => sandbox/src/test/org/apache/lucene/sandbox/facet/plain/histograms}/TestHistogramCollectorManager.java (98%) diff --git a/lucene/facet/src/java/module-info.java b/lucene/facet/src/java/module-info.java index 0dfaea136d61..2aa1e3e494e5 100644 --- a/lucene/facet/src/java/module-info.java +++ b/lucene/facet/src/java/module-info.java @@ -20,7 +20,6 @@ requires org.apache.lucene.core; exports org.apache.lucene.facet; - exports org.apache.lucene.facet.histogram; exports org.apache.lucene.facet.range; exports org.apache.lucene.facet.sortedset; exports org.apache.lucene.facet.taxonomy; diff --git a/lucene/sandbox/src/java/module-info.java b/lucene/sandbox/src/java/module-info.java index f40a05af433a..f3fd6199ba3c 100644 --- a/lucene/sandbox/src/java/module-info.java +++ b/lucene/sandbox/src/java/module-info.java @@ -34,6 +34,7 @@ exports org.apache.lucene.sandbox.facet.iterators; exports org.apache.lucene.sandbox.facet.cutters; exports org.apache.lucene.sandbox.facet.labels; + exports org.apache.lucene.sandbox.facet.plain.histograms; provides org.apache.lucene.codecs.PostingsFormat with org.apache.lucene.sandbox.codecs.idversion.IDVersionPostingsFormat; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java similarity index 99% rename from lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java rename to lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java index f443db6b1f84..7d929b0aa0cf 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollector.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.facet.histogram; +package org.apache.lucene.sandbox.facet.plain.histograms; import java.io.IOException; import org.apache.lucene.index.DocValues; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java similarity index 88% rename from lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java rename to lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java index e14de49d3b4c..4346ad2c2e32 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/histogram/HistogramCollectorManager.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.facet.histogram; +package org.apache.lucene.sandbox.facet.plain.histograms; import java.io.IOException; import java.util.Collection; @@ -27,8 +27,11 @@ /** * {@link CollectorManager} that computes a histogram of the distribution of the values of a field. * - *

The returned {@link LongIntHashMap} maps quotients to the number of documents whose value - * returns this number when divided by the given {@code interval}. + *

It takes an {@code interval} as a parameter and counts the number of documents that fall into + * intervals [0, interval), [interval, 2*interval), etc. The keys of the returned {@link + * LongIntHashMap} identify these intervals as the quotient of the integer division by {@code + * interval}. Said otherwise, a key equal to {@code k} maps to values in the interval {@code [k * + * interval, (k+1) * interval)}. * *

This implementation is optimized for the case when {@code field} is part of the index sort and * has a {@link FieldType#setDocValuesSkipIndexType skip index}. diff --git a/lucene/facet/src/java/org/apache/lucene/facet/histogram/package-info.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/package-info.java similarity index 93% rename from lucene/facet/src/java/org/apache/lucene/facet/histogram/package-info.java rename to lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/package-info.java index f2dfbc1dc74e..7a4ce5a1f222 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/histogram/package-info.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/package-info.java @@ -16,4 +16,4 @@ */ /** Provides histotgram faceting capabilities. */ -package org.apache.lucene.facet.histogram; +package org.apache.lucene.sandbox.facet.plain.histograms; diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/package-info.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/package-info.java new file mode 100644 index 000000000000..d25a59ddeb9f --- /dev/null +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides plain faceting capabilities, as opposed to {@link + * org.apache.lucene.sandbox.facet.cutters cutters} and {@link + * org.apache.lucene.sandbox.facet.recorders recorders}, which allow composing faceting components + * together. + */ +package org.apache.lucene.sandbox.facet.plain; diff --git a/lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/facet/plain/histograms/TestHistogramCollectorManager.java similarity index 98% rename from lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java rename to lucene/sandbox/src/test/org/apache/lucene/sandbox/facet/plain/histograms/TestHistogramCollectorManager.java index d121825a31ac..da62537b18ba 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/histogram/TestHistogramCollectorManager.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/facet/plain/histograms/TestHistogramCollectorManager.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.facet.histogram; +package org.apache.lucene.sandbox.facet.plain.histograms; import java.io.IOException; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; From 85bde38f7539679fcc29f16607a9fc04fde8a902 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Feb 2025 11:46:33 +0100 Subject: [PATCH 6/8] interval -> intervalWidth --- .../plain/histograms/HistogramCollector.java | 49 ++++++++++--------- .../histograms/HistogramCollectorManager.java | 22 ++++----- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java index 7d929b0aa0cf..a4c722c49179 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java @@ -34,13 +34,13 @@ final class HistogramCollector implements Collector { private final String field; - private final long interval; + private final long intervalWidth; private final int maxBuckets; private final LongIntHashMap counts; - HistogramCollector(String field, long interval, int maxBuckets) { + HistogramCollector(String field, long intervalWidth, int maxBuckets) { this.field = field; - this.interval = interval; + this.intervalWidth = intervalWidth; this.maxBuckets = maxBuckets; this.counts = new LongIntHashMap(); } @@ -60,19 +60,20 @@ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOExcept SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); NumericDocValues singleton = DocValues.unwrapSingleton(values); if (singleton == null) { - return new HistogramNaiveLeafCollector(values, interval, maxBuckets, counts); + return new HistogramNaiveLeafCollector(values, intervalWidth, maxBuckets, counts); } else { DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); if (skipper != null) { - long leafMinQuotient = Math.floorDiv(skipper.minValue(), interval); - long leafMaxQuotient = Math.floorDiv(skipper.maxValue(), interval); + long leafMinQuotient = Math.floorDiv(skipper.minValue(), intervalWidth); + long leafMaxQuotient = Math.floorDiv(skipper.maxValue(), intervalWidth); if (leafMaxQuotient - leafMinQuotient <= 1024) { // Only use the optimized implementation if there is a small number of unique quotients, // so that we can count them using a dense array instead of a hash table. - return new HistogramLeafCollector(singleton, skipper, interval, maxBuckets, counts); + return new HistogramLeafCollector(singleton, skipper, intervalWidth, maxBuckets, counts); } } - return new HistogramNaiveSingleValuedLeafCollector(singleton, interval, maxBuckets, counts); + return new HistogramNaiveSingleValuedLeafCollector( + singleton, intervalWidth, maxBuckets, counts); } } @@ -92,14 +93,14 @@ LongIntHashMap getCounts() { private static class HistogramNaiveLeafCollector implements LeafCollector { private final SortedNumericDocValues values; - private final long interval; + private final long intervalWidth; private final int maxBuckets; private final LongIntHashMap counts; HistogramNaiveLeafCollector( - SortedNumericDocValues values, long interval, int maxBuckets, LongIntHashMap counts) { + SortedNumericDocValues values, long intervalWidth, int maxBuckets, LongIntHashMap counts) { this.values = values; - this.interval = interval; + this.intervalWidth = intervalWidth; this.maxBuckets = maxBuckets; this.counts = counts; } @@ -114,7 +115,7 @@ public void collect(int doc) throws IOException { long prevQuotient = Long.MIN_VALUE; for (int i = 0; i < valueCount; ++i) { final long value = values.nextValue(); - final long quotient = Math.floorDiv(value, interval); + final long quotient = Math.floorDiv(value, intervalWidth); // We must not double-count values that divide to the same quotient since this returns doc // counts as opposed to value counts. if (quotient != prevQuotient) { @@ -134,14 +135,14 @@ public void collect(int doc) throws IOException { private static class HistogramNaiveSingleValuedLeafCollector implements LeafCollector { private final NumericDocValues values; - private final long interval; + private final long intervalWidth; private final int maxBuckets; private final LongIntHashMap counts; HistogramNaiveSingleValuedLeafCollector( - NumericDocValues values, long interval, int maxBuckets, LongIntHashMap counts) { + NumericDocValues values, long intervalWidth, int maxBuckets, LongIntHashMap counts) { this.values = values; - this.interval = interval; + this.intervalWidth = intervalWidth; this.maxBuckets = maxBuckets; this.counts = counts; } @@ -153,7 +154,7 @@ public void setScorer(Scorable scorer) throws IOException {} public void collect(int doc) throws IOException { if (values.advanceExact(doc)) { final long value = values.longValue(); - final long quotient = Math.floorDiv(value, interval); + final long quotient = Math.floorDiv(value, intervalWidth); counts.addTo(quotient, 1); checkMaxBuckets(counts.size(), maxBuckets); } @@ -168,7 +169,7 @@ private static class HistogramLeafCollector implements LeafCollector { private final NumericDocValues values; private final DocValuesSkipper skipper; - private final long interval; + private final long intervalWidth; private final int maxBuckets; private final int[] counts; private final long leafMinQuotient; @@ -188,17 +189,17 @@ private static class HistogramLeafCollector implements LeafCollector { HistogramLeafCollector( NumericDocValues values, DocValuesSkipper skipper, - long interval, + long intervalWidth, int maxBuckets, LongIntHashMap collectorCounts) { this.values = values; this.skipper = skipper; - this.interval = interval; + this.intervalWidth = intervalWidth; this.maxBuckets = maxBuckets; this.collectorCounts = collectorCounts; - leafMinQuotient = Math.floorDiv(skipper.minValue(), interval); - long leafMaxQuotient = Math.floorDiv(skipper.maxValue(), interval); + leafMinQuotient = Math.floorDiv(skipper.minValue(), intervalWidth); + long leafMaxQuotient = Math.floorDiv(skipper.maxValue(), intervalWidth); counts = new int[Math.toIntExact(leafMaxQuotient - leafMinQuotient + 1)]; } @@ -223,8 +224,8 @@ private void advanceSkipper(int doc) throws IOException { // Now find the highest level where all docs have the same quotient. for (int level = 0; level < skipper.numLevels(); ++level) { int totalDocsAtLevel = skipper.maxDocID(level) - skipper.minDocID(level) + 1; - long minQuotient = Math.floorDiv(skipper.minValue(level), interval); - long maxQuotient = Math.floorDiv(skipper.maxValue(level), interval); + long minQuotient = Math.floorDiv(skipper.minValue(level), intervalWidth); + long maxQuotient = Math.floorDiv(skipper.maxValue(level), intervalWidth); if (skipper.docCount(level) == totalDocsAtLevel && minQuotient == maxQuotient) { // All docs at this level have a value, and all values map to the same quotient. @@ -247,7 +248,7 @@ public void collect(int doc) throws IOException { counts[upToQuotientIndex]++; } else if (values.advanceExact(doc)) { final long value = values.longValue(); - final long quotient = Math.floorDiv(value, interval); + final long quotient = Math.floorDiv(value, intervalWidth); counts[(int) (quotient - leafMinQuotient)]++; } } diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java index 4346ad2c2e32..5c7e73930ce4 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java @@ -27,11 +27,11 @@ /** * {@link CollectorManager} that computes a histogram of the distribution of the values of a field. * - *

It takes an {@code interval} as a parameter and counts the number of documents that fall into - * intervals [0, interval), [interval, 2*interval), etc. The keys of the returned {@link - * LongIntHashMap} identify these intervals as the quotient of the integer division by {@code - * interval}. Said otherwise, a key equal to {@code k} maps to values in the interval {@code [k * - * interval, (k+1) * interval)}. + *

It takes an {@code intervalWidth} as a parameter and counts the number of documents that fall + * into intervals [0, intervalWidth), [intervalWidth, 2*intervalWidth), etc. The keys of the + * returned {@link LongIntHashMap} identify these intervals as the quotient of the integer division + * by {@code intervalWidth}. Said otherwise, a key equal to {@code k} maps to values in the interval + * {@code [k * intervalWidth, (k+1) * intervalWidth)}. * *

This implementation is optimized for the case when {@code field} is part of the index sort and * has a {@link FieldType#setDocValuesSkipIndexType skip index}. @@ -48,7 +48,7 @@ public final class HistogramCollectorManager private static final int DEFAULT_MAX_BUCKETS = 1024; private final String field; - private final long interval; + private final long intervalWidth; private final int maxBuckets; /** @@ -66,12 +66,12 @@ public HistogramCollectorManager(String field, long interval) { * @param maxBuckets Max allowed number of buckets. Note that this is checked at runtime and on a * best-effort basis. */ - public HistogramCollectorManager(String field, long interval, int maxBuckets) { + public HistogramCollectorManager(String field, long intervalWidth, int maxBuckets) { this.field = Objects.requireNonNull(field); - if (interval < 2) { - throw new IllegalArgumentException("interval must be at least 2, got: " + interval); + if (intervalWidth < 2) { + throw new IllegalArgumentException("intervalWidth must be at least 2, got: " + intervalWidth); } - this.interval = interval; + this.intervalWidth = intervalWidth; if (maxBuckets < 1) { throw new IllegalArgumentException("maxBuckets must be at least 1, got: " + maxBuckets); } @@ -80,7 +80,7 @@ public HistogramCollectorManager(String field, long interval, int maxBuckets) { @Override public HistogramCollector newCollector() throws IOException { - return new HistogramCollector(field, interval, maxBuckets); + return new HistogramCollector(field, intervalWidth, maxBuckets); } @Override From 29183e4370629f282b85bd18d86e68f701d123bb Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Feb 2025 11:54:02 +0100 Subject: [PATCH 7/8] quotient -> bucket --- .../plain/histograms/HistogramCollector.java | 71 +++++++++---------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java index a4c722c49179..5a35be5773d7 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java @@ -64,11 +64,12 @@ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOExcept } else { DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); if (skipper != null) { - long leafMinQuotient = Math.floorDiv(skipper.minValue(), intervalWidth); - long leafMaxQuotient = Math.floorDiv(skipper.maxValue(), intervalWidth); - if (leafMaxQuotient - leafMinQuotient <= 1024) { - // Only use the optimized implementation if there is a small number of unique quotients, - // so that we can count them using a dense array instead of a hash table. + long leafMinBucket = Math.floorDiv(skipper.minValue(), intervalWidth); + long leafMaxBucket = Math.floorDiv(skipper.maxValue(), intervalWidth); + if (leafMaxBucket - leafMinBucket <= 1024) { + // Only use the optimized implementation if there is a small number of unique buckets, + // so that we can count them using a dense array instead of a hash table. This helps save + // the overhead of hashing and collision resolution. return new HistogramLeafCollector(singleton, skipper, intervalWidth, maxBuckets, counts); } } @@ -112,16 +113,16 @@ public void setScorer(Scorable scorer) throws IOException {} public void collect(int doc) throws IOException { if (values.advanceExact(doc)) { int valueCount = values.docValueCount(); - long prevQuotient = Long.MIN_VALUE; + long prevBucket = Long.MIN_VALUE; for (int i = 0; i < valueCount; ++i) { final long value = values.nextValue(); - final long quotient = Math.floorDiv(value, intervalWidth); - // We must not double-count values that divide to the same quotient since this returns doc + final long bucket = Math.floorDiv(value, intervalWidth); + // We must not double-count values that map to the same bucket since this returns doc // counts as opposed to value counts. - if (quotient != prevQuotient) { - counts.addTo(quotient, 1); + if (bucket != prevBucket) { + counts.addTo(bucket, 1); checkMaxBuckets(counts.size(), maxBuckets); - prevQuotient = quotient; + prevBucket = bucket; } } } @@ -154,8 +155,8 @@ public void setScorer(Scorable scorer) throws IOException {} public void collect(int doc) throws IOException { if (values.advanceExact(doc)) { final long value = values.longValue(); - final long quotient = Math.floorDiv(value, intervalWidth); - counts.addTo(quotient, 1); + final long bucket = Math.floorDiv(value, intervalWidth); + counts.addTo(bucket, 1); checkMaxBuckets(counts.size(), maxBuckets); } } @@ -172,19 +173,17 @@ private static class HistogramLeafCollector implements LeafCollector { private final long intervalWidth; private final int maxBuckets; private final int[] counts; - private final long leafMinQuotient; + private final long leafMinBucket; private final LongIntHashMap collectorCounts; - /** - * Max doc ID (inclusive) up to which all docs may map to values that have the same quotient. - */ + /** Max doc ID (inclusive) up to which all docs values may map to the same bucket. */ private int upToInclusive = -1; - /** Whether all docs up to {@link #upToInclusive} map to values that have the same quotient. */ - private boolean upToSameQuotient; + /** Whether all docs up to {@link #upToInclusive} values map to the same bucket. */ + private boolean upToSameBucket; /** Index in {@link #counts} for docs up to {@link #upToInclusive}. */ - private int upToQuotientIndex; + private int upToBucketIndex; HistogramLeafCollector( NumericDocValues values, @@ -198,9 +197,9 @@ private static class HistogramLeafCollector implements LeafCollector { this.maxBuckets = maxBuckets; this.collectorCounts = collectorCounts; - leafMinQuotient = Math.floorDiv(skipper.minValue(), intervalWidth); - long leafMaxQuotient = Math.floorDiv(skipper.maxValue(), intervalWidth); - counts = new int[Math.toIntExact(leafMaxQuotient - leafMinQuotient + 1)]; + leafMinBucket = Math.floorDiv(skipper.minValue(), intervalWidth); + long leafMaxBucket = Math.floorDiv(skipper.maxValue(), intervalWidth); + counts = new int[Math.toIntExact(leafMaxBucket - leafMinBucket + 1)]; } @Override @@ -210,7 +209,7 @@ private void advanceSkipper(int doc) throws IOException { if (doc > skipper.maxDocID(0)) { skipper.advance(doc); } - upToSameQuotient = false; + upToSameBucket = false; if (skipper.minDocID(0) > doc) { // Corner case which happens if `doc` doesn't have a value and is between two intervals of @@ -221,17 +220,17 @@ private void advanceSkipper(int doc) throws IOException { upToInclusive = skipper.maxDocID(0); - // Now find the highest level where all docs have the same quotient. + // Now find the highest level where all docs map to the same bucket. for (int level = 0; level < skipper.numLevels(); ++level) { int totalDocsAtLevel = skipper.maxDocID(level) - skipper.minDocID(level) + 1; - long minQuotient = Math.floorDiv(skipper.minValue(level), intervalWidth); - long maxQuotient = Math.floorDiv(skipper.maxValue(level), intervalWidth); + long minBucket = Math.floorDiv(skipper.minValue(level), intervalWidth); + long maxBucket = Math.floorDiv(skipper.maxValue(level), intervalWidth); - if (skipper.docCount(level) == totalDocsAtLevel && minQuotient == maxQuotient) { - // All docs at this level have a value, and all values map to the same quotient. + if (skipper.docCount(level) == totalDocsAtLevel && minBucket == maxBucket) { + // All docs at this level have a value, and all values map to the same bucket. upToInclusive = skipper.maxDocID(level); - upToSameQuotient = true; - upToQuotientIndex = (int) (minQuotient - this.leafMinQuotient); + upToSameBucket = true; + upToBucketIndex = (int) (minBucket - this.leafMinBucket); } else { break; } @@ -244,12 +243,12 @@ public void collect(int doc) throws IOException { advanceSkipper(doc); } - if (upToSameQuotient) { - counts[upToQuotientIndex]++; + if (upToSameBucket) { + counts[upToBucketIndex]++; } else if (values.advanceExact(doc)) { final long value = values.longValue(); - final long quotient = Math.floorDiv(value, intervalWidth); - counts[(int) (quotient - leafMinQuotient)]++; + final long bucket = Math.floorDiv(value, intervalWidth); + counts[(int) (bucket - leafMinBucket)]++; } } @@ -257,7 +256,7 @@ public void collect(int doc) throws IOException { public void finish() throws IOException { // Put counts that we computed in the int[] back into the hash map. for (int i = 0; i < counts.length; ++i) { - collectorCounts.addTo(leafMinQuotient + i, counts[i]); + collectorCounts.addTo(leafMinBucket + i, counts[i]); } checkMaxBuckets(collectorCounts.size(), maxBuckets); } From 78eaa4e8cfb3f39e704b8dd3f7c93365c630d2b6 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 11 Feb 2025 11:57:38 +0100 Subject: [PATCH 8/8] intervalWidth -> bucketWidth --- .../plain/histograms/HistogramCollector.java | 48 +++++++++---------- .../histograms/HistogramCollectorManager.java | 28 +++++------ 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java index 5a35be5773d7..7c3bbb26530a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollector.java @@ -34,13 +34,13 @@ final class HistogramCollector implements Collector { private final String field; - private final long intervalWidth; + private final long bucketWidth; private final int maxBuckets; private final LongIntHashMap counts; - HistogramCollector(String field, long intervalWidth, int maxBuckets) { + HistogramCollector(String field, long bucketWidth, int maxBuckets) { this.field = field; - this.intervalWidth = intervalWidth; + this.bucketWidth = bucketWidth; this.maxBuckets = maxBuckets; this.counts = new LongIntHashMap(); } @@ -60,21 +60,21 @@ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOExcept SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); NumericDocValues singleton = DocValues.unwrapSingleton(values); if (singleton == null) { - return new HistogramNaiveLeafCollector(values, intervalWidth, maxBuckets, counts); + return new HistogramNaiveLeafCollector(values, bucketWidth, maxBuckets, counts); } else { DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); if (skipper != null) { - long leafMinBucket = Math.floorDiv(skipper.minValue(), intervalWidth); - long leafMaxBucket = Math.floorDiv(skipper.maxValue(), intervalWidth); + long leafMinBucket = Math.floorDiv(skipper.minValue(), bucketWidth); + long leafMaxBucket = Math.floorDiv(skipper.maxValue(), bucketWidth); if (leafMaxBucket - leafMinBucket <= 1024) { // Only use the optimized implementation if there is a small number of unique buckets, // so that we can count them using a dense array instead of a hash table. This helps save // the overhead of hashing and collision resolution. - return new HistogramLeafCollector(singleton, skipper, intervalWidth, maxBuckets, counts); + return new HistogramLeafCollector(singleton, skipper, bucketWidth, maxBuckets, counts); } } return new HistogramNaiveSingleValuedLeafCollector( - singleton, intervalWidth, maxBuckets, counts); + singleton, bucketWidth, maxBuckets, counts); } } @@ -94,14 +94,14 @@ LongIntHashMap getCounts() { private static class HistogramNaiveLeafCollector implements LeafCollector { private final SortedNumericDocValues values; - private final long intervalWidth; + private final long bucketWidth; private final int maxBuckets; private final LongIntHashMap counts; HistogramNaiveLeafCollector( - SortedNumericDocValues values, long intervalWidth, int maxBuckets, LongIntHashMap counts) { + SortedNumericDocValues values, long bucketWidth, int maxBuckets, LongIntHashMap counts) { this.values = values; - this.intervalWidth = intervalWidth; + this.bucketWidth = bucketWidth; this.maxBuckets = maxBuckets; this.counts = counts; } @@ -116,7 +116,7 @@ public void collect(int doc) throws IOException { long prevBucket = Long.MIN_VALUE; for (int i = 0; i < valueCount; ++i) { final long value = values.nextValue(); - final long bucket = Math.floorDiv(value, intervalWidth); + final long bucket = Math.floorDiv(value, bucketWidth); // We must not double-count values that map to the same bucket since this returns doc // counts as opposed to value counts. if (bucket != prevBucket) { @@ -136,14 +136,14 @@ public void collect(int doc) throws IOException { private static class HistogramNaiveSingleValuedLeafCollector implements LeafCollector { private final NumericDocValues values; - private final long intervalWidth; + private final long bucketWidth; private final int maxBuckets; private final LongIntHashMap counts; HistogramNaiveSingleValuedLeafCollector( - NumericDocValues values, long intervalWidth, int maxBuckets, LongIntHashMap counts) { + NumericDocValues values, long bucketWidth, int maxBuckets, LongIntHashMap counts) { this.values = values; - this.intervalWidth = intervalWidth; + this.bucketWidth = bucketWidth; this.maxBuckets = maxBuckets; this.counts = counts; } @@ -155,7 +155,7 @@ public void setScorer(Scorable scorer) throws IOException {} public void collect(int doc) throws IOException { if (values.advanceExact(doc)) { final long value = values.longValue(); - final long bucket = Math.floorDiv(value, intervalWidth); + final long bucket = Math.floorDiv(value, bucketWidth); counts.addTo(bucket, 1); checkMaxBuckets(counts.size(), maxBuckets); } @@ -170,7 +170,7 @@ private static class HistogramLeafCollector implements LeafCollector { private final NumericDocValues values; private final DocValuesSkipper skipper; - private final long intervalWidth; + private final long bucketWidth; private final int maxBuckets; private final int[] counts; private final long leafMinBucket; @@ -188,17 +188,17 @@ private static class HistogramLeafCollector implements LeafCollector { HistogramLeafCollector( NumericDocValues values, DocValuesSkipper skipper, - long intervalWidth, + long bucketWidth, int maxBuckets, LongIntHashMap collectorCounts) { this.values = values; this.skipper = skipper; - this.intervalWidth = intervalWidth; + this.bucketWidth = bucketWidth; this.maxBuckets = maxBuckets; this.collectorCounts = collectorCounts; - leafMinBucket = Math.floorDiv(skipper.minValue(), intervalWidth); - long leafMaxBucket = Math.floorDiv(skipper.maxValue(), intervalWidth); + leafMinBucket = Math.floorDiv(skipper.minValue(), bucketWidth); + long leafMaxBucket = Math.floorDiv(skipper.maxValue(), bucketWidth); counts = new int[Math.toIntExact(leafMaxBucket - leafMinBucket + 1)]; } @@ -223,8 +223,8 @@ private void advanceSkipper(int doc) throws IOException { // Now find the highest level where all docs map to the same bucket. for (int level = 0; level < skipper.numLevels(); ++level) { int totalDocsAtLevel = skipper.maxDocID(level) - skipper.minDocID(level) + 1; - long minBucket = Math.floorDiv(skipper.minValue(level), intervalWidth); - long maxBucket = Math.floorDiv(skipper.maxValue(level), intervalWidth); + long minBucket = Math.floorDiv(skipper.minValue(level), bucketWidth); + long maxBucket = Math.floorDiv(skipper.maxValue(level), bucketWidth); if (skipper.docCount(level) == totalDocsAtLevel && minBucket == maxBucket) { // All docs at this level have a value, and all values map to the same bucket. @@ -247,7 +247,7 @@ public void collect(int doc) throws IOException { counts[upToBucketIndex]++; } else if (values.advanceExact(doc)) { final long value = values.longValue(); - final long bucket = Math.floorDiv(value, intervalWidth); + final long bucket = Math.floorDiv(value, bucketWidth); counts[(int) (bucket - leafMinBucket)]++; } } diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java index 5c7e73930ce4..e22aff3d0406 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/facet/plain/histograms/HistogramCollectorManager.java @@ -27,11 +27,11 @@ /** * {@link CollectorManager} that computes a histogram of the distribution of the values of a field. * - *

It takes an {@code intervalWidth} as a parameter and counts the number of documents that fall - * into intervals [0, intervalWidth), [intervalWidth, 2*intervalWidth), etc. The keys of the - * returned {@link LongIntHashMap} identify these intervals as the quotient of the integer division - * by {@code intervalWidth}. Said otherwise, a key equal to {@code k} maps to values in the interval - * {@code [k * intervalWidth, (k+1) * intervalWidth)}. + *

It takes an {@code bucketWidth} as a parameter and counts the number of documents that fall + * into intervals [0, bucketWidth), [bucketWidth, 2*bucketWidth), etc. The keys of the returned + * {@link LongIntHashMap} identify these intervals as the quotient of the integer division by {@code + * bucketWidth}. Said otherwise, a key equal to {@code k} maps to values in the interval {@code [k * + * bucketWidth, (k+1) * bucketWidth)}. * *

This implementation is optimized for the case when {@code field} is part of the index sort and * has a {@link FieldType#setDocValuesSkipIndexType skip index}. @@ -48,16 +48,16 @@ public final class HistogramCollectorManager private static final int DEFAULT_MAX_BUCKETS = 1024; private final String field; - private final long intervalWidth; + private final long bucketWidth; private final int maxBuckets; /** * Compute a histogram of the distribution of the values of the given {@code field} according to - * the given {@code interval}. This configures a maximum number of buckets equal to the default of + * the given {@code bucketWidth}. This configures a maximum number of buckets equal to the default of * 1024. */ - public HistogramCollectorManager(String field, long interval) { - this(field, interval, DEFAULT_MAX_BUCKETS); + public HistogramCollectorManager(String field, long bucketWidth) { + this(field, bucketWidth, DEFAULT_MAX_BUCKETS); } /** @@ -66,12 +66,12 @@ public HistogramCollectorManager(String field, long interval) { * @param maxBuckets Max allowed number of buckets. Note that this is checked at runtime and on a * best-effort basis. */ - public HistogramCollectorManager(String field, long intervalWidth, int maxBuckets) { + public HistogramCollectorManager(String field, long bucketWidth, int maxBuckets) { this.field = Objects.requireNonNull(field); - if (intervalWidth < 2) { - throw new IllegalArgumentException("intervalWidth must be at least 2, got: " + intervalWidth); + if (bucketWidth < 2) { + throw new IllegalArgumentException("bucketWidth must be at least 2, got: " + bucketWidth); } - this.intervalWidth = intervalWidth; + this.bucketWidth = bucketWidth; if (maxBuckets < 1) { throw new IllegalArgumentException("maxBuckets must be at least 1, got: " + maxBuckets); } @@ -80,7 +80,7 @@ public HistogramCollectorManager(String field, long intervalWidth, int maxBucket @Override public HistogramCollector newCollector() throws IOException { - return new HistogramCollector(field, intervalWidth, maxBuckets); + return new HistogramCollector(field, bucketWidth, maxBuckets); } @Override