iter

Author: john-wagster

lucene/CHANGES.txt

@@ -75,8 +75,12 @@ Improvements
   mergeFactor segments together when the merge is below the min merge size.
   (Adrien Grand)
 
+* GITHUB#14154: Add UnwrappingReuseStrategy for AnalyzerWrapper that consults
+  the wrapped analyzer's strategy to decide if components can be reused or need
+  to be updated. (Mayya Sharipova)
+
 * GITHUB#14192: Added support for RegExp to handle case insensitive matching
-  across the full set of Unicode characters  
+  across the full set of Unicode characters. (John Wagster)
 
 Optimizations
 ---------------------
@@ -89,7 +93,11 @@ Optimizations
 * GITHUB#14133: Dense blocks of postings are now encoded as bit sets.
   (Adrien Grand)
 
-# GITHUB#14169: Optimize ContextQuery with big number of contexts. (Mayya Sharipova)
+* GITHUB#14169: Optimize ContextQuery with big number of contexts. (Mayya Sharipova)
+
+* GITHUB#14181: Add updateable random scorer interface for knn vector index building. This allows
+  for fewer objects to be created during indexing and simplifies internally used iterfaces.
+  (Ben Trent)
 
 Bug Fixes
 ---------------------

lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerBenchmark.java

@@ -35,6 +35,7 @@
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 import org.openjdk.jmh.annotations.*;
 
 @BenchmarkMode(Mode.Throughput)
@@ -57,7 +58,7 @@ public class VectorScorerBenchmark {
   IndexInput in;
   KnnVectorValues vectorValues;
   byte[] vec1, vec2;
-  RandomVectorScorer scorer;
+  UpdateableRandomVectorScorer scorer;
 
   @Setup(Level.Iteration)
   public void init() throws IOException {
@@ -76,7 +77,8 @@ public void init() throws IOException {
     scorer =
         FlatVectorScorerUtil.getLucene99FlatVectorsScorer()
             .getRandomVectorScorerSupplier(DOT_PRODUCT, vectorValues)
-            .scorer(0);
+            .scorer();
+    scorer.setScoringOrdinal(0);
   }
 
   @TearDown

lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/FlatBitVectorsScorer.java

@@ -26,14 +26,14 @@
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 
 /** A bit vector scorer for scoring byte vectors. */
 public class FlatBitVectorsScorer implements FlatVectorsScorer {
   @Override
   public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
       VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)
       throws IOException {
-    assert vectorValues instanceof ByteVectorValues;
     if (vectorValues instanceof ByteVectorValues byteVectorValues) {
       return new BitRandomVectorScorerSupplier(byteVectorValues);
     }
@@ -51,14 +51,13 @@ public RandomVectorScorer getRandomVectorScorer(
   public RandomVectorScorer getRandomVectorScorer(
       VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, byte[] target)
       throws IOException {
-    assert vectorValues instanceof ByteVectorValues;
     if (vectorValues instanceof ByteVectorValues byteVectorValues) {
       return new BitRandomVectorScorer(byteVectorValues, target);
     }
     throw new IllegalArgumentException("vectorValues must be an instance of ByteVectorValues");
   }
 
-  static class BitRandomVectorScorer implements RandomVectorScorer {
+  static class BitRandomVectorScorer implements UpdateableRandomVectorScorer {
     private final ByteVectorValues vectorValues;
     private final int bitDimensions;
     private final byte[] query;
@@ -80,6 +79,11 @@ public int maxOrd() {
       return vectorValues.size();
     }
 
+    @Override
+    public void setScoringOrdinal(int node) throws IOException {
+      System.arraycopy(vectorValues.vectorValue(node), 0, query, 0, query.length);
+    }
+
     @Override
     public int ordToDoc(int ord) {
       return vectorValues.ordToDoc(ord);
@@ -93,24 +97,22 @@ public Bits getAcceptOrds(Bits acceptDocs) {
 
   static class BitRandomVectorScorerSupplier implements RandomVectorScorerSupplier {
     protected final ByteVectorValues vectorValues;
-    protected final ByteVectorValues vectorValues1;
-    protected final ByteVectorValues vectorValues2;
+    protected final ByteVectorValues targetVectors;
 
     public BitRandomVectorScorerSupplier(ByteVectorValues vectorValues) throws IOException {
       this.vectorValues = vectorValues;
-      this.vectorValues1 = vectorValues.copy();
-      this.vectorValues2 = vectorValues.copy();
+      this.targetVectors = vectorValues.copy();
     }
 
     @Override
-    public RandomVectorScorer scorer(int ord) throws IOException {
-      byte[] query = vectorValues1.vectorValue(ord);
-      return new BitRandomVectorScorer(vectorValues2, query);
+    public UpdateableRandomVectorScorer scorer() throws IOException {
+      byte[] query = new byte[vectorValues.dimension()];
+      return new BitRandomVectorScorer(vectorValues, query);
     }
 
     @Override
     public RandomVectorScorerSupplier copy() throws IOException {
-      return new BitRandomVectorScorerSupplier(vectorValues.copy());
+      return new BitRandomVectorScorerSupplier(vectorValues);
     }
   }
 

lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java

@@ -342,7 +342,7 @@ public void close() {
    * TokenFilter} which also serves as the {@link TokenStream} returned by {@link
    * Analyzer#tokenStream(String, Reader)}.
    */
-  public static final class TokenStreamComponents {
+  public static class TokenStreamComponents {
     /** Original source of the tokens. */
     protected final Consumer<Reader> source;
 

lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java

@@ -41,7 +41,6 @@
  * @since 4.0.0
  */
 public abstract class AnalyzerWrapper extends Analyzer {
-
   /**
    * Creates a new AnalyzerWrapper with the given reuse strategy.
    *
@@ -53,7 +52,10 @@ public abstract class AnalyzerWrapper extends Analyzer {
    * @see #getReuseStrategy()
    */
   protected AnalyzerWrapper(ReuseStrategy reuseStrategy) {
-    super(reuseStrategy);
+    super(
+        reuseStrategy instanceof DelegatingAnalyzerWrapper.DelegatingReuseStrategy
+            ? reuseStrategy
+            : new UnwrappingReuseStrategy(reuseStrategy));
   }
 
   /**
@@ -117,7 +119,10 @@ protected Reader wrapReaderForNormalization(String fieldName, Reader reader) {
 
   @Override
   protected final TokenStreamComponents createComponents(String fieldName) {
-    return wrapComponents(fieldName, getWrappedAnalyzer(fieldName).createComponents(fieldName));
+    TokenStreamComponents wrappedComponents =
+        getWrappedAnalyzer(fieldName).createComponents(fieldName);
+    TokenStreamComponents wrapperComponents = wrapComponents(fieldName, wrappedComponents);
+    return new TokenStreamComponentsWrapper(wrapperComponents, wrappedComponents);
   }
 
   @Override
@@ -151,4 +156,63 @@ protected final Reader initReaderForNormalization(String fieldName, Reader reade
   protected final AttributeFactory attributeFactory(String fieldName) {
     return getWrappedAnalyzer(fieldName).attributeFactory(fieldName);
   }
+
+  /**
+   * A {@link org.apache.lucene.analysis.Analyzer.ReuseStrategy} that checks the wrapped analyzer's
+   * strategy for reusability. If the wrapped analyzer's strategy returns null, components need to
+   * be re-created.
+   */
+  public static final class UnwrappingReuseStrategy extends ReuseStrategy {
+    private final ReuseStrategy reuseStrategy;
+
+    public UnwrappingReuseStrategy(ReuseStrategy reuseStrategy) {
+      this.reuseStrategy = reuseStrategy;
+    }
+
+    @Override
+    public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {
+      if (analyzer instanceof AnalyzerWrapper wrapper) {
+        final Analyzer wrappedAnalyzer = wrapper.getWrappedAnalyzer(fieldName);
+        if (wrappedAnalyzer.getReuseStrategy().getReusableComponents(wrappedAnalyzer, fieldName)
+            == null) {
+          return null;
+        }
+      }
+      return reuseStrategy.getReusableComponents(analyzer, fieldName);
+    }
+
+    @Override
+    public void setReusableComponents(
+        Analyzer analyzer, String fieldName, TokenStreamComponents components) {
+      reuseStrategy.setReusableComponents(analyzer, fieldName, components);
+
+      if (analyzer instanceof AnalyzerWrapper wrapper) {
+        assert components instanceof TokenStreamComponentsWrapper;
+        final TokenStreamComponentsWrapper wrapperComponents =
+            (TokenStreamComponentsWrapper) components;
+        final Analyzer wrappedAnalyzer = wrapper.getWrappedAnalyzer(fieldName);
+        wrappedAnalyzer
+            .getReuseStrategy()
+            .setReusableComponents(
+                wrappedAnalyzer, fieldName, wrapperComponents.getWrappedComponents());
+      }
+    }
+  }
+
+  /**
+   * A {@link Analyzer.TokenStreamComponents} that decorates the wrapper with access to the wrapped
+   * components.
+   */
+  static final class TokenStreamComponentsWrapper extends TokenStreamComponents {
+    private final TokenStreamComponents wrapped;
+
+    TokenStreamComponentsWrapper(TokenStreamComponents wrapper, TokenStreamComponents wrapped) {
+      super(wrapper.getSource(), wrapper.getTokenStream());
+      this.wrapped = wrapped;
+    }
+
+    TokenStreamComponents getWrappedComponents() {
+      return wrapped;
+    }
+  }
 }

lucene/core/src/java/org/apache/lucene/analysis/DelegatingAnalyzerWrapper.java

@@ -69,7 +69,11 @@ protected final Reader wrapReaderForNormalization(String fieldName, Reader reade
     return super.wrapReaderForNormalization(fieldName, reader);
   }
 
-  private static final class DelegatingReuseStrategy extends ReuseStrategy {
+  /**
+   * A {@link org.apache.lucene.analysis.Analyzer.ReuseStrategy} that delegates to the wrapped
+   * analyzer's strategy for reusability of components.
+   */
+  static final class DelegatingReuseStrategy extends ReuseStrategy {
     DelegatingAnalyzerWrapper wrapper;
     private final ReuseStrategy fallbackStrategy;
 

lucene/core/src/java/org/apache/lucene/analysis/package-info.java

@@ -343,7 +343,7 @@
  *       undefined as long as they both leave from the same position. As result, all synonyms of a
  *       token would be considered to appear in exactly the same position as that token, and so
  *       would they be seen by phrase and proximity searches. For multi-token synonyms to work
- *       correctly, you should use {@code SynoymGraphFilter} at search time only.
+ *       correctly, you should use {@code SynonymGraphFilter} at search time only.
  * </ol>
  *
  * <h3>Token Position Length</h3>

lucene/core/src/java/org/apache/lucene/codecs/hnsw/DefaultFlatVectorScorer.java

@@ -24,6 +24,7 @@
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 
 /**
  * Default implementation of {@link FlatVectorsScorer}.
@@ -89,24 +90,29 @@ public String toString() {
   /** RandomVectorScorerSupplier for bytes vector */
   private static final class ByteScoringSupplier implements RandomVectorScorerSupplier {
     private final ByteVectorValues vectors;
-    private final ByteVectorValues vectors1;
-    private final ByteVectorValues vectors2;
+    private final ByteVectorValues targetVectors;
     private final VectorSimilarityFunction similarityFunction;
 
     private ByteScoringSupplier(
         ByteVectorValues vectors, VectorSimilarityFunction similarityFunction) throws IOException {
       this.vectors = vectors;
-      vectors1 = vectors.copy();
-      vectors2 = vectors.copy();
+      targetVectors = vectors.copy();
       this.similarityFunction = similarityFunction;
     }
 
     @Override
-    public RandomVectorScorer scorer(int ord) {
-      return new RandomVectorScorer.AbstractRandomVectorScorer(vectors) {
+    public UpdateableRandomVectorScorer scorer() throws IOException {
+      byte[] vector = new byte[vectors.dimension()];
+      return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(vectors) {
+
+        @Override
+        public void setScoringOrdinal(int node) throws IOException {
+          System.arraycopy(targetVectors.vectorValue(node), 0, vector, 0, vector.length);
+        }
+
         @Override
         public float score(int node) throws IOException {
-          return similarityFunction.compare(vectors1.vectorValue(ord), vectors2.vectorValue(node));
+          return similarityFunction.compare(vector, targetVectors.vectorValue(node));
         }
       };
     }
@@ -125,24 +131,28 @@ public String toString() {
   /** RandomVectorScorerSupplier for Float vector */
   private static final class FloatScoringSupplier implements RandomVectorScorerSupplier {
     private final FloatVectorValues vectors;
-    private final FloatVectorValues vectors1;
-    private final FloatVectorValues vectors2;
+    private final FloatVectorValues targetVectors;
     private final VectorSimilarityFunction similarityFunction;
 
     private FloatScoringSupplier(
         FloatVectorValues vectors, VectorSimilarityFunction similarityFunction) throws IOException {
       this.vectors = vectors;
-      vectors1 = vectors.copy();
-      vectors2 = vectors.copy();
+      targetVectors = vectors.copy();
       this.similarityFunction = similarityFunction;
     }
 
     @Override
-    public RandomVectorScorer scorer(int ord) {
-      return new RandomVectorScorer.AbstractRandomVectorScorer(vectors) {
+    public UpdateableRandomVectorScorer scorer() throws IOException {
+      float[] vector = new float[vectors.dimension()];
+      return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(vectors) {
         @Override
         public float score(int node) throws IOException {
-          return similarityFunction.compare(vectors1.vectorValue(ord), vectors2.vectorValue(node));
+          return similarityFunction.compare(vector, targetVectors.vectorValue(node));
+        }
+
+        @Override
+        public void setScoringOrdinal(int node) throws IOException {
+          System.arraycopy(targetVectors.vectorValue(node), 0, vector, 0, vector.length);
         }
       };
     }

lucene/core/src/java/org/apache/lucene/codecs/hnsw/ScalarQuantizedVectorScorer.java

@@ -24,6 +24,7 @@
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizedVectorSimilarity;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
@@ -147,11 +148,18 @@ private ScalarQuantizedRandomVectorScorerSupplier(
     }
 
     @Override
-    public RandomVectorScorer scorer(int ord) throws IOException {
+    public UpdateableRandomVectorScorer scorer() throws IOException {
       final QuantizedByteVectorValues vectorsCopy = values.copy();
-      final byte[] queryVector = values.vectorValue(ord);
-      final float queryOffset = values.getScoreCorrectionConstant(ord);
-      return new RandomVectorScorer.AbstractRandomVectorScorer(vectorsCopy) {
+      byte[] queryVector = new byte[values.dimension()];
+      return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(vectorsCopy) {
+        float queryOffset = 0;
+
+        @Override
+        public void setScoringOrdinal(int node) throws IOException {
+          System.arraycopy(vectorsCopy.vectorValue(node), 0, queryVector, 0, queryVector.length);
+          queryOffset = vectorsCopy.getScoreCorrectionConstant(node);
+        }
+
         @Override
         public float score(int node) throws IOException {
           byte[] nodeVector = vectorsCopy.vectorValue(node);

lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java

@@ -52,8 +52,8 @@
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;
-import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 
 /**
  * Writes vector values to index segments.
@@ -507,8 +507,8 @@ static final class FlatCloseableRandomVectorScorerSupplier
     }
 
     @Override
-    public RandomVectorScorer scorer(int ord) throws IOException {
-      return supplier.scorer(ord);
+    public UpdateableRandomVectorScorer scorer() throws IOException {
+      return supplier.scorer();
     }
 
     @Override