Add a mode of error handling without exceptions and ctest support

Adding a mode of error handling that does not rely on C++ exceptions,
with Abseil-like Status and StatusOr return types (but not adding a
dependency on Abseil). The old exceptions-based API is still available,
and the new functions are suffixed with "NoExceptions".

Refactoring the CMake build system. Organizing examples and tests and
reducing repetition. Adding the HNSWLIB_ENABLE_EXCEPTIONS option (ON by
default).

Also adding ctest support.

Author: michaelbautin

CMakeLists.txt

@@ -6,6 +6,56 @@ project(hnswlib
 include(GNUInstallDirs)
 include(CheckCXXCompilerFlag)
 
+# These example/test targets catch exceptions, so exceptions should always be
+# enabled building these files even if they are disabled in other targets.
+# We check that each target included in this list is a real target.
+set(HNSWLIB_TARGETS_REQUIRING_EXCEPTIONS
+    example_mt_filter
+    example_mt_replace_deleted
+    example_mt_search
+    multiThread_replace_test
+    test_updates)
+
+# Adds an example or test target. The target name parameter is followed by
+# the list of source files. Automatically links with the hnswlib library.
+# Also decides whether to enable exceptions when building the target. 
+# If HNSWLIB_ENABLE_EXCEPTIONS is ON, exceptions are always enabled.
+# If HNSWLIB_ENABLE_EXCEPTIONS is OFF, exceptions are only enabled for the
+# specific targets listed in HNSWLIB_TARGETS_REQUIRING_EXCEPTIONS.
+function(add_example_or_test TARGET_NAME ...)
+    add_executable(${ARGV})
+    target_link_libraries(${TARGET_NAME} hnswlib)
+    list(FIND HNSWLIB_TARGETS_REQUIRING_EXCEPTIONS "${TARGET_NAME}" found_at_index)
+    if(found_at_index GREATER -1)
+        if(NOT HNSWLIB_ENABLE_EXCEPTIONS)
+            message("Enabling exceptions for target ${TARGET_NAME} as a special case")
+        endif()
+        set(should_enable_exceptions ON)
+    else()
+        set(should_enable_exceptions "${HNSWLIB_ENABLE_EXCEPTIONS}")
+    endif()
+    if(should_enable_exceptions)
+        target_compile_options("${TARGET_NAME}" PUBLIC -fexceptions)
+    else()
+        target_compile_options("${TARGET_NAME}" PUBLIC -fno-exceptions)
+    endif()
+    if(NOT ${TARGET_NAME} MATCHES "^(main|test_updates)$")
+        add_test(
+            NAME ${TARGET_NAME}
+            COMMAND ${TARGET_NAME}
+        )
+    endif()
+endfunction()
+
+option(HNSWLIB_ENABLE_EXCEPTIONS "Whether to enable exceptions in hnswlib" ON)
+if(HNSWLIB_ENABLE_EXCEPTIONS)
+    message("Exceptions are enabled using HNSWLIB_ENABLE_EXCEPTIONS=ON (default)")
+else()
+    message("Exceptions are disabled using HNSWLIB_ENABLE_EXCEPTIONS=OFF")
+endif()
+
+set(CMAKE_CXX_STANDARD 11)
+
 add_library(hnswlib INTERFACE)
 add_library(hnswlib::hnswlib ALIAS hnswlib)
 
@@ -28,14 +78,14 @@ install(EXPORT hnswlibTargets
 # Examples and tests
 if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     option(HNSWLIB_EXAMPLES "Build examples and tests." ON)
+    message("Building examples and tests")
 else()
     option(HNSWLIB_EXAMPLES "Build examples and tests." OFF)
 endif()
 if(HNSWLIB_EXAMPLES)
-    set(CMAKE_CXX_STANDARD 11)
-
+    enable_testing()
     if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-      SET( CMAKE_CXX_FLAGS  "-Ofast -std=c++11 -DHAVE_CXX0X -openmp -fpic -ftree-vectorize" )
+      SET( CMAKE_CXX_FLAGS  "-Ofast -DHAVE_CXX0X -openmp -fpic -ftree-vectorize" )
       check_cxx_compiler_flag("-march=native" COMPILER_SUPPORT_NATIVE_FLAG)
       if(COMPILER_SUPPORT_NATIVE_FLAG)
         SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native" )
@@ -48,58 +98,48 @@ if(HNSWLIB_EXAMPLES)
         endif()
       endif()
     elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-      SET( CMAKE_CXX_FLAGS  "-Ofast -lrt -std=c++11 -DHAVE_CXX0X -march=native -fpic -w -fopenmp -ftree-vectorize -ftree-vectorizer-verbose=0" )
+      SET( CMAKE_CXX_FLAGS  "-Ofast -lrt -DHAVE_CXX0X -march=native -fpic -w -fopenmp -ftree-vectorize -ftree-vectorizer-verbose=0" )
     elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
       SET( CMAKE_CXX_FLAGS  "/O2 -DHAVE_CXX0X /W1 /openmp /EHsc" )
     endif()
 
-    # examples
-    add_executable(example_search examples/cpp/example_search.cpp)
-    target_link_libraries(example_search hnswlib)
-
-    add_executable(example_epsilon_search examples/cpp/example_epsilon_search.cpp)
-    target_link_libraries(example_epsilon_search hnswlib)
-
-    add_executable(example_multivector_search examples/cpp/example_multivector_search.cpp)
-    target_link_libraries(example_multivector_search hnswlib)
-
-    add_executable(example_filter examples/cpp/example_filter.cpp)
-    target_link_libraries(example_filter hnswlib)
-
-    add_executable(example_replace_deleted examples/cpp/example_replace_deleted.cpp)
-    target_link_libraries(example_replace_deleted hnswlib)
-
-    add_executable(example_mt_search examples/cpp/example_mt_search.cpp)
-    target_link_libraries(example_mt_search hnswlib)
-
-    add_executable(example_mt_filter examples/cpp/example_mt_filter.cpp)
-    target_link_libraries(example_mt_filter hnswlib)
-
-    add_executable(example_mt_replace_deleted examples/cpp/example_mt_replace_deleted.cpp)
-    target_link_libraries(example_mt_replace_deleted hnswlib)
-
-    # tests
-    add_executable(multivector_search_test tests/cpp/multivector_search_test.cpp)
-    target_link_libraries(multivector_search_test hnswlib)
-
-    add_executable(epsilon_search_test tests/cpp/epsilon_search_test.cpp)
-    target_link_libraries(epsilon_search_test hnswlib)
-
-    add_executable(test_updates tests/cpp/updates_test.cpp)
-    target_link_libraries(test_updates hnswlib)
-
-    add_executable(searchKnnCloserFirst_test tests/cpp/searchKnnCloserFirst_test.cpp)
-    target_link_libraries(searchKnnCloserFirst_test hnswlib)
-
-    add_executable(searchKnnWithFilter_test tests/cpp/searchKnnWithFilter_test.cpp)
-    target_link_libraries(searchKnnWithFilter_test hnswlib)
-
-    add_executable(multiThreadLoad_test tests/cpp/multiThreadLoad_test.cpp)
-    target_link_libraries(multiThreadLoad_test hnswlib)
-
-    add_executable(multiThread_replace_test tests/cpp/multiThread_replace_test.cpp)
-    target_link_libraries(multiThread_replace_test hnswlib)
-
-    add_executable(main tests/cpp/main.cpp tests/cpp/sift_1b.cpp)
-    target_link_libraries(main hnswlib)
+    set(EXAMPLE_NAMES
+        example_epsilon_search
+        example_filter
+        example_mt_filter
+        example_mt_replace_deleted
+        example_mt_search
+        example_multivector_search
+        example_replace_deleted
+        example_search)
+
+    foreach(example_name IN LISTS EXAMPLE_NAMES)
+      add_example_or_test("${example_name}" "examples/cpp/${example_name}.cpp")
+    endforeach()
+
+    set(TEST_NAMES
+        multivector_search_test
+        epsilon_search_test
+        searchKnnCloserFirst_test
+        searchKnnWithFilter_test
+        multiThreadLoad_test
+        multiThread_replace_test)
+    foreach(test_name IN LISTS TEST_NAMES)
+      add_example_or_test("${test_name}" "tests/cpp/${test_name}.cpp")
+    endforeach()
+
+    # This test deviates from the above pattern of naming test executables.
+    add_example_or_test(test_updates tests/cpp/updates_test.cpp)
+
+    # For historical reasons, the "main" program links with sift_1b.cpp.
+    add_example_or_test(main tests/cpp/main.cpp tests/cpp/sift_1b.cpp)
+
+    foreach(target_name IN LISTS HNSWLIB_TARGETS_REQUIRING_EXCEPTIONS)
+        if(NOT TARGET ${target_name})
+            message(FATAL_ERROR
+                    "Target '${target_name}' included in "
+                    "HNSWLIB_TARGETS_REQUIRING_EXCEPTIONS does not exist. "
+                    "Please check if this is a typo.")
+        endif()
+    endforeach()
 endif()

examples/cpp/example_epsilon_search.cpp

@@ -53,7 +53,7 @@ int main() {
         }
         std::cout << "Query #" << i << "\n";
         hnswlib::EpsilonSearchStopCondition<dist_t> stop_condition(epsilon2, min_num_candidates, max_elements);
-        std::vector<std::pair<float, hnswlib::labeltype>> result = 
+        std::vector<std::pair<float, hnswlib::labeltype>> result =
             alg_hnsw->searchStopConditionClosest(query_data, stop_condition);
         size_t num_vectors = result.size();
         std::cout << "Found " << num_vectors << " vectors\n";

examples/cpp/example_mt_replace_deleted.cpp

@@ -69,7 +69,7 @@ int main() {
     int num_threads = 20;       // Number of threads for operations with index
 
     // Initing index with allow_replace_deleted=true
-    int seed = 100; 
+    int seed = 100;
     hnswlib::L2Space space(dim);
     hnswlib::HierarchicalNSW<float>* alg_hnsw = new hnswlib::HierarchicalNSW<float>(&space, max_elements, M, ef_construction, seed, true);
 

examples/cpp/example_multivector_search.cpp

@@ -63,7 +63,7 @@ int main() {
         }
         std::cout << "Query #" << i << "\n";
         hnswlib::MultiVectorSearchStopCondition<docidtype, dist_t> stop_condition(space, num_docs, ef_collection);
-        std::vector<std::pair<float, hnswlib::labeltype>> result = 
+        std::vector<std::pair<float, hnswlib::labeltype>> result =
             alg_hnsw->searchStopConditionClosest(query_data, stop_condition);
         size_t num_vectors = result.size();
 

examples/cpp/example_replace_deleted.cpp

@@ -9,7 +9,7 @@ int main() {
     int ef_construction = 200;  // Controls index search speed/build speed tradeoff
 
     // Initing index with allow_replace_deleted=true
-    int seed = 100; 
+    int seed = 100;
     hnswlib::L2Space space(dim);
     hnswlib::HierarchicalNSW<float>* alg_hnsw = new hnswlib::HierarchicalNSW<float>(&space, max_elements, M, ef_construction, seed, true);
 

hnswlib/bruteforce.h

@@ -1,4 +1,7 @@
 #pragma once
+
+#include "hnswlib.h"
+
 #include <unordered_map>
 #include <fstream>
 #include <mutex>
@@ -51,7 +54,7 @@ class BruteforceSearch : public AlgorithmInterface<dist_t> {
         size_per_element_ = data_size_ + sizeof(labeltype);
         data_ = (char *) malloc(maxElements * size_per_element_);
         if (data_ == nullptr)
-            throw std::runtime_error("Not enough memory: BruteforceSearch failed to allocate data");
+            HNSWLIB_THROW_RUNTIME_ERROR("Not enough memory: BruteforceSearch failed to allocate data");
         cur_element_count = 0;
     }
 
@@ -61,7 +64,7 @@ class BruteforceSearch : public AlgorithmInterface<dist_t> {
     }
 
 
-    void addPoint(const void *datapoint, labeltype label, bool replace_deleted = false) {
+    Status addPointNoExceptions(const void *datapoint, labeltype label, bool replace_deleted = false) override {
         int idx;
         {
             std::unique_lock<std::mutex> lock(index_lock);
@@ -71,7 +74,7 @@ class BruteforceSearch : public AlgorithmInterface<dist_t> {
                 idx = search->second;
             } else {
                 if (cur_element_count >= maxelements_) {
-                    throw std::runtime_error("The number of elements exceeds the specified limit\n");
+                    return Status("The number of elements exceeds the specified limit");
                 }
                 idx = cur_element_count;
                 dict_external_to_internal[label] = idx;
@@ -80,6 +83,7 @@ class BruteforceSearch : public AlgorithmInterface<dist_t> {
         }
         memcpy(data_ + size_per_element_ * idx + data_size_, &label, sizeof(labeltype));
         memcpy(data_ + size_per_element_ * idx, datapoint, data_size_);
+        return OkStatus();
     }
 
 
@@ -103,8 +107,9 @@ class BruteforceSearch : public AlgorithmInterface<dist_t> {
     }
 
 
-    std::priority_queue<std::pair<dist_t, labeltype >>
-    searchKnn(const void *query_data, size_t k, BaseFilterFunctor* isIdAllowed = nullptr) const {
+    using DistanceLabelPriorityQueue = typename AlgorithmInterface<dist_t>::DistanceLabelPriorityQueue;
+    StatusOr<DistanceLabelPriorityQueue>
+    searchKnnNoExceptions(const void *query_data, size_t k, BaseFilterFunctor* isIdAllowed = nullptr) const override {
         assert(k <= cur_element_count);
         std::priority_queue<std::pair<dist_t, labeltype >> topResults;
         dist_t lastdist = std::numeric_limits<dist_t>::max();
@@ -153,7 +158,7 @@ class BruteforceSearch : public AlgorithmInterface<dist_t> {
         size_per_element_ = data_size_ + sizeof(labeltype);
         data_ = (char *) malloc(maxelements_ * size_per_element_);
         if (data_ == nullptr)
-            throw std::runtime_error("Not enough memory: loadIndex failed to allocate data");
+            HNSWLIB_THROW_RUNTIME_ERROR("Not enough memory: loadIndex failed to allocate data");
 
         input.read(data_, maxelements_ * size_per_element_);