Add a Faiss codec for KNN searches

gradle/generation/extract-jdk-apis.gradle

@@ -19,9 +19,23 @@ def resources = scriptResources(buildscript)
 
 configure(project(":lucene:core")) {
   ext {
-    apijars = layout.projectDirectory.dir("src/generated/jdk")
     mrjarJavaVersions = [ 21 ]
   }
+}
+
+configure(project(":lucene:sandbox")) {
+  ext {
+    mrjarJavaVersions = [ 22 ]
+  }
+}
+
+configure([
+        project(":lucene:core"),
+        project(":lucene:sandbox")
+]) {
+  ext {
+    apijars = layout.projectDirectory.dir("src/generated/jdk")
+  }
 
   configurations {
     apiextractor

gradle/generation/extract-jdk-apis/ExtractJdkApis.java

@@ -54,7 +54,8 @@ public final class ExtractJdkApis {
   private static final String PATTERN_VECTOR_VM_INTERNALS = "java.base/jdk/internal/vm/vector/VectorSupport{,$Vector,$VectorMask,$VectorPayload,$VectorShuffle}";
 
   static final Map<Integer,List<String>> CLASSFILE_PATTERNS = Map.of(
-      21, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR)
+      21, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR),
+      22, List.of(PATTERN_PANAMA_FOREIGN)
   );
 
   public static void main(String... args) throws IOException {

gradle/generation/regenerate.gradle

@@ -91,6 +91,7 @@ configure([
         project(":lucene:queryparser"),
         project(":lucene:expressions"),
         project(":lucene:test-framework"),
+        project(":lucene:sandbox"),
 ]) {
     task regenerate() {
       description "Rerun any code or static data generation tasks."

gradle/java/core-mrjar.gradle

@@ -17,7 +17,10 @@
 
 // Produce an MR-JAR with Java 19+ foreign and vector implementations
 
-configure(project(":lucene:core")) {
+configure([
+        project(":lucene:core"),
+        project(":lucene:sandbox")
+]) {
   plugins.withType(JavaPlugin) {
     mrjarJavaVersions.each { jdkVersion ->
       sourceSets.create("main${jdkVersion}") {

lucene/sandbox/src/java/module-info.java

@@ -20,8 +20,10 @@
   requires org.apache.lucene.core;
   requires org.apache.lucene.queries;
   requires org.apache.lucene.facet;
+  requires java.logging;
 
   exports org.apache.lucene.payloads;
+  exports org.apache.lucene.sandbox.codecs.faiss;
   exports org.apache.lucene.sandbox.codecs.idversion;
   exports org.apache.lucene.sandbox.codecs.quantization;
   exports org.apache.lucene.sandbox.document;
@@ -37,4 +39,6 @@
 
   provides org.apache.lucene.codecs.PostingsFormat with
       org.apache.lucene.sandbox.codecs.idversion.IDVersionPostingsFormat;
+  provides org.apache.lucene.codecs.KnnVectorsFormat with
+      org.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormatProvider;
 }

lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormatProvider.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.faiss;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.util.Arrays;
+import java.util.logging.Logger;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.KnnVectorsWriter;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+/**
+ * Wraps <a href="https://github.com/facebookresearch/faiss">Faiss</a> to create and search vector
+ * indexes. This class is mainly for backwards compatibility with older versions of Java (&lt;22),
+ * use underlying format directly after upgrade.
+ *
+ * @lucene.experimental
+ */
+public class FaissKnnVectorsFormatProvider extends KnnVectorsFormat {
+  private final KnnVectorsFormat delegate;
+
+  public FaissKnnVectorsFormatProvider() {
+    this(new Object[0]);
+  }
+
+  public FaissKnnVectorsFormatProvider(Object... args) {
+    super(FaissKnnVectorsFormatProvider.class.getSimpleName());
+
+    KnnVectorsFormat delegate;
+    try {
+      Class<?> cls =
+          MethodHandles.lookup()
+              .findClass("org.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormat");
+
+      MethodType methodType =
+          MethodType.methodType(
+              void.class, Arrays.stream(args).map(Object::getClass).toArray(Class<?>[]::new));
+
+      delegate =
+          (KnnVectorsFormat)
+              MethodHandles.lookup().findConstructor(cls, methodType).invokeWithArguments(args);
+
+    } catch (
+        @SuppressWarnings("unused")
+        ClassNotFoundException e) {
+
+      delegate = null;
+      Logger.getLogger(getClass().getName())
+          .warning("FaissKnnVectorsFormat class missing, this object is unusable!");
+
+    } catch (NoSuchMethodException | IllegalAccessException e) {
+      throw new LinkageError("FaissKnnVectorsFormat is missing correctly typed constructor", e);
+    } catch (Throwable t) {
+      throw new RuntimeException(t);
+    }
+    this.delegate = delegate;
+  }
+
+  @Override
+  public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+    return delegate.fieldsWriter(state);
+  }
+
+  @Override
+  public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
+    return delegate.fieldsReader(state);
+  }
+
+  @Override
+  public int getMaxDimensions(String fieldName) {
+    return delegate.getMaxDimensions(fieldName);
+  }
+}

lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Wraps <a href="https://github.com/facebookresearch/faiss">Faiss</a> to create and search vector
+ * indexes via {@link org.apache.lucene.sandbox.codecs.faiss.FaissKnnVectorsFormatProvider}.
+ */
+package org.apache.lucene.sandbox.codecs.faiss;

lucene/sandbox/src/java22/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.faiss;
+
+import java.io.IOException;
+import java.util.Locale;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.KnnVectorsWriter;
+import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
+import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+public final class FaissKnnVectorsFormat extends KnnVectorsFormat {
+  public static final String NAME = FaissKnnVectorsFormat.class.getSimpleName();
+  static final int VERSION_START = 0;
+  static final int VERSION_CURRENT = VERSION_START;
+  static final String META_CODEC_NAME = NAME + "Meta";
+  static final String DATA_CODEC_NAME = NAME + "Data";
+  static final String META_EXTENSION = "faissm";
+  static final String DATA_EXTENSION = "faissd";
+
+  private final String description;
+  private final String indexParams;
+  private final KnnVectorsFormat rawVectorsFormat;
+
+  @SuppressWarnings("unused")
+  public FaissKnnVectorsFormat() {
+    this("IDMap,HNSW32", "efConstruction=200");
+  }
+
+  public FaissKnnVectorsFormat(String description, String indexParams) {
+    super(NAME);
+    this.description = description;
+    this.indexParams = indexParams;
+    this.rawVectorsFormat =
+        new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer());
+  }
+
+  @Override
+  public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+    return new FaissKnnVectorsWriter(
+        description, indexParams, state, rawVectorsFormat.fieldsWriter(state));
+  }
+
+  @Override
+  public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
+    return new FaissKnnVectorsReader(state, rawVectorsFormat.fieldsReader(state));
+  }
+
+  @Override
+  public int getMaxDimensions(String fieldName) {
+    return DEFAULT_MAX_DIMENSIONS;
+  }
+
+  @Override
+  public String toString() {
+    return String.format(
+        Locale.ROOT, "%s(description=%s indexParams=%s)", NAME, description, indexParams);
+  }
+}