diff --git a/cmake/Env.cmake b/cmake/Env.cmake index 5f70306e9..d6a17fb5c 100644 --- a/cmake/Env.cmake +++ b/cmake/Env.cmake @@ -256,6 +256,7 @@ elseif(WIN32) ob_define(OB_VCPKG_DIR "C:/VcpkgInstalled/x64-windows") ob_define(OB_OPENSSL_DIR "C:/Program Files/OpenSSL-Win64") ob_define(OB_LLVM_DIR "C:/Program Files/LLVM18") + ob_define(OB_VSAG_DIR "${DEP_3RD_DIR}/vsag") elseif(UNIX) # NO RELERO: -Wl,-znorelro # Partial RELRO: -Wl,-z,relro diff --git a/cmake/Pack.cmake b/cmake/Pack.cmake index 5268983a1..db39ecacb 100644 --- a/cmake/Pack.cmake +++ b/cmake/Pack.cmake @@ -47,6 +47,7 @@ if(WIN32) # its runtime libraries into the MSI package. set(_SEEKDB_EXE "${CMAKE_BINARY_DIR}/src/observer/seekdb.exe") set(_VCPKG_BIN_DIR "${OB_VCPKG_DIR}/bin") + set(_VSAG_BIN_DIR "${OB_VSAG_DIR}/bin") file(WRITE "${CMAKE_BINARY_DIR}/_bundle_dlls.cmake.in" [=[ file(GET_RUNTIME_DEPENDENCIES @@ -57,35 +58,44 @@ file(GET_RUNTIME_DEPENDENCIES CONFLICTING_DEPENDENCIES_PREFIX _conflicts DIRECTORIES "@_VCPKG_BIN_DIR@" + "@_VSAG_BIN_DIR@" PRE_EXCLUDE_REGEXES "^api-ms-" "^ext-ms-" ) -set(_vcpkg_bin "@_VCPKG_BIN_DIR@") +set(_search_dirs "@_VCPKG_BIN_DIR@;@_VSAG_BIN_DIR@") set(_bundled 0) -# Install resolved dependencies that have a vcpkg copy. -# System-only DLLs (KERNEL32, ADVAPI32, ...) are absent from vcpkg and -# therefore skipped — they ship with every Windows installation. +# Install resolved dependencies that live in vcpkg or vsag directories. +# System-only DLLs (KERNEL32, ADVAPI32, ...) are absent from these dirs +# and therefore skipped — they ship with every Windows installation. foreach(_file ${_resolved}) get_filename_component(_name "${_file}" NAME) - if(EXISTS "${_vcpkg_bin}/${_name}") - message(STATUS " ${_name}") - file(INSTALL DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" - TYPE SHARED_LIBRARY FILES "${_vcpkg_bin}/${_name}") - math(EXPR _bundled "${_bundled} + 1") - endif() + set(_found FALSE) + foreach(_dir ${_search_dirs}) + if(EXISTS "${_dir}/${_name}") + message(STATUS " ${_name}") + file(INSTALL DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" + TYPE SHARED_LIBRARY FILES "${_dir}/${_name}") + math(EXPR _bundled "${_bundled} + 1") + set(_found TRUE) + break() + endif() + endforeach() endforeach() -# Conflicting dependencies (same DLL in vcpkg AND System32, e.g. libssl). +# Conflicting dependencies (same DLL in multiple dirs AND System32). foreach(_name ${_conflicts_FILENAMES}) - if(EXISTS "${_vcpkg_bin}/${_name}") - message(STATUS " ${_name} (conflict resolved -> vcpkg)") - file(INSTALL DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" - TYPE SHARED_LIBRARY FILES "${_vcpkg_bin}/${_name}") - math(EXPR _bundled "${_bundled} + 1") - endif() + foreach(_dir ${_search_dirs}) + if(EXISTS "${_dir}/${_name}") + message(STATUS " ${_name} (conflict resolved -> ${_dir})") + file(INSTALL DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" + TYPE SHARED_LIBRARY FILES "${_dir}/${_name}") + math(EXPR _bundled "${_bundled} + 1") + break() + endif() + endforeach() endforeach() message(STATUS "Bundled ${_bundled} runtime DLLs into bin/") diff --git a/deps/init/dep_create.ps1 b/deps/init/dep_create.ps1 index 780ff15dd..118b82834 100644 --- a/deps/init/dep_create.ps1 +++ b/deps/init/dep_create.ps1 @@ -9,6 +9,7 @@ After extraction the layout is: deps/3rd/vcpkg/x64-windows/ (vcpkg installed packages) deps/3rd/openssl/ (OpenSSL) + deps/3rd/vsag/ (vsag vector search library) deps/3rd/tools/cmake/ (CMake) deps/3rd/tools/ninja/ (Ninja) deps/3rd/tools/llvm18/ (LLVM 18) @@ -159,6 +160,7 @@ Write-Log "" Write-Log "Layout:" Write-Log " deps/3rd/vcpkg/x64-windows/ vcpkg packages" Write-Log " deps/3rd/openssl/ OpenSSL" +Write-Log " deps/3rd/vsag/ vsag vector search library" Write-Log " deps/3rd/tools/cmake/ CMake" Write-Log " deps/3rd/tools/ninja/ Ninja" Write-Log " deps/3rd/tools/llvm18/ LLVM 18" diff --git a/deps/init/oceanbase.windows.x86_64.deps b/deps/init/oceanbase.windows.x86_64.deps index aea5d180e..22b6113ad 100644 --- a/deps/init/oceanbase.windows.x86_64.deps +++ b/deps/init/oceanbase.windows.x86_64.deps @@ -6,6 +6,7 @@ repo=https://mirrors.oceanbase.com/oceanbase/development-kit/windows/11/x86_64 [deps] devdeps-vcpkg-all-20260414.tar.gz devdeps-openssl-20260414.tar.gz +devdeps-vsag-20260416.tar.gz [tools] obdevtools-cmake-20260414.tar.gz diff --git a/deps/init/pack_win_deps.ps1 b/deps/init/pack_win_deps.ps1 index cc8e016c2..6433ddeb5 100644 --- a/deps/init/pack_win_deps.ps1 +++ b/deps/init/pack_win_deps.ps1 @@ -36,6 +36,7 @@ param( [string]$VcpkgRoot = "C:\VcpkgInstalled", [string]$OpenSSLDir = "C:\Program Files\OpenSSL-Win64", [string]$LLVMDir = "C:\Program Files\LLVM18", + [string]$VsagDir = "C:\VsagInstalled", [string]$DateStamp = (Get-Date -Format "yyyyMMdd") ) @@ -218,6 +219,18 @@ if ($FlexBisonDir -and (Test-Path $FlexBisonDir)) { if ($result) { $toolsPackages += $result } } +# 7. vsag (vector search library) +if (Test-Path $VsagDir) { + $name = "devdeps-vsag-$DateStamp.tar.gz" + Write-Log "Packing vsag from $VsagDir ..." + $result = New-DepArchive -ArchiveName $name -TopDirName "devdeps-vsag-$DateStamp" -Mappings @{ + "vsag" = $VsagDir + } + if ($result) { $depsPackages += $result } +} else { + Write-Err "vsag not found: $VsagDir" +} + # -- Generate deps file ---------------------------------------------- $depsFile = "$ScriptDir\oceanbase.windows.x86_64.deps" $content = @" diff --git a/deps/oblib/src/CMakeLists.txt b/deps/oblib/src/CMakeLists.txt index d658f08be..14c6dedc7 100644 --- a/deps/oblib/src/CMakeLists.txt +++ b/deps/oblib/src/CMakeLists.txt @@ -42,6 +42,7 @@ target_include_directories( $<$:${VCPKG_INCLUDE_DIR}/mysql/mariadb> $<$:${VCPKG_INCLUDE_DIR}/mysql/mysql> $<$:${SQLITE_COMPAT_DIR}> + $<$:${OB_VSAG_DIR}/include> ${DEVTOOLS_DIR} ${DEVTOOLS_DIR}/include ${DEP_DIR}/include/apr-1/ diff --git a/deps/oblib/src/lib/CMakeLists.txt b/deps/oblib/src/lib/CMakeLists.txt index 64f8859eb..c9d1453c7 100644 --- a/deps/oblib/src/lib/CMakeLists.txt +++ b/deps/oblib/src/lib/CMakeLists.txt @@ -568,7 +568,21 @@ elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") libfmt.a ) elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") - message(STATUS "Need to support Windows architecture: ${CMAKE_SYSTEM_PROCESSOR}") + target_link_directories(oblib_lib PUBLIC ${OB_VSAG_DIR}/lib) + target_include_directories(oblib_lib PUBLIC ${OB_VSAG_DIR}/include) + target_link_libraries(oblib_lib + PUBLIC + cpuinfo.lib + simd.lib + vsag_static.lib + diskann.lib + openblas.lib + libomp.lib + roaring.lib + antlr4-runtime-static.lib + antlr4-autogen.lib + fmt.lib + ) else() message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}") endif() diff --git a/deps/oblib/src/lib/vector/ob_vector_util.cpp b/deps/oblib/src/lib/vector/ob_vector_util.cpp index 31e81851a..94c1e7bb0 100644 --- a/deps/oblib/src/lib/vector/ob_vector_util.cpp +++ b/deps/oblib/src/lib/vector/ob_vector_util.cpp @@ -16,12 +16,6 @@ #define USING_LOG_PREFIX LIB -#ifdef _WIN32 -#ifndef OB_BUILD_CDC_DISABLE_VSAG -#define OB_BUILD_CDC_DISABLE_VSAG -#endif -#endif - #include "ob_vector_util.h" #include "lib/string/ob_string.h" @@ -131,7 +125,7 @@ int create_index(obvsag::VectorIndexPtr &index_handler, int index_type, const ch INIT_SUCC(ret); #ifdef OB_BUILD_CDC_DISABLE_VSAG return ret; -#else +#else obvsag::set_block_size_limit(2*1024*1024); LOG_INFO("vector index create params: ", K(index_type), KCSTRING(dtype), KCSTRING(metric), K(use_reorder), K(doc_prune_ratio), K(window_size), KP(allocator), K(extra_info_size)); return obvsag::create_index(index_handler, static_cast(index_type), @@ -298,8 +292,8 @@ int knn_search(obvsag::VectorIndexPtr index_handler, uint32_t len, uint32_t *dim return ret; #else return obvsag::knn_search(index_handler, len, dims, vals, topk, - result_dist, result_ids, extra_info, result_size, - query_prune_ratio, n_candidate, + result_dist, result_ids, extra_info, result_size, + query_prune_ratio, n_candidate, invalid, reverse_filter, is_extra_info_filter, valid_ratio, allocator, need_extra_info); #endif diff --git a/deps/oblib/src/lib/vector/ob_vector_util.h b/deps/oblib/src/lib/vector/ob_vector_util.h index f157b6be1..19c05637c 100644 --- a/deps/oblib/src/lib/vector/ob_vector_util.h +++ b/deps/oblib/src/lib/vector/ob_vector_util.h @@ -17,37 +17,9 @@ #ifndef OB_VECTOR_UTIL_H #define OB_VECTOR_UTIL_H #include -#ifdef _WIN32 -#include -#include -#include -namespace vsag { - class Logger { - public: - enum Level { TRACE, DEBUG, INFO, WARN, ERROR, CRITICAL, OFF }; - virtual void SetLevel(Level level) = 0; - virtual void Trace(const std::string& msg) = 0; - virtual void Debug(const std::string& msg) = 0; - virtual void Info(const std::string& msg) = 0; - virtual void Warn(const std::string& msg) = 0; - virtual void Error(const std::string& msg) = 0; - virtual void Critical(const std::string& msg) = 0; - virtual ~Logger() = default; - }; - class Allocator { - public: - virtual std::string Name() = 0; - virtual void* Allocate(uint64_t size) = 0; - virtual void Deallocate(void* p) = 0; - virtual void* Reallocate(void* p, uint64_t size) = 0; - virtual ~Allocator() = default; - }; -} -#else #include #include #include -#endif #include #include "lib/allocator/page_arena.h" #include "lib/vector/ob_vsag_adaptor.h" diff --git a/deps/oblib/src/lib/vector/ob_vsag_adaptor.cpp b/deps/oblib/src/lib/vector/ob_vsag_adaptor.cpp index 726dde49e..140f00254 100644 --- a/deps/oblib/src/lib/vector/ob_vsag_adaptor.cpp +++ b/deps/oblib/src/lib/vector/ob_vsag_adaptor.cpp @@ -16,12 +16,6 @@ #define USING_LOG_PREFIX LIB -#ifdef _WIN32 -#ifndef OB_BUILD_CDC_DISABLE_VSAG -#define OB_BUILD_CDC_DISABLE_VSAG -#endif -#endif - #include "ob_vsag_adaptor.h" #include #ifndef OB_BUILD_CDC_DISABLE_VSAG @@ -332,7 +326,7 @@ int HnswIndexHandler::get_vid_bound(int64_t &min_vid, int64_t &max_vid) uint64_t HnswIndexHandler::estimate_memory(const uint64_t row_count, const bool is_build) { - + uint64_t size = 0; if (IPIVF_TYPE == index_type_) { // TODO(ningxin.ning): use vsag EstimateMemory @@ -706,7 +700,7 @@ int construct_vsag_create_param( return ret; } -int construct_vsag_sindi_create_param(uint8_t create_type, const char *dtype, const char *metric, +int construct_vsag_sindi_create_param(uint8_t create_type, const char *dtype, const char *metric, void *allocator, int extra_info_size, bool use_reorder, float doc_prune_ratio, int window_size, char *result_param_str) { @@ -796,7 +790,7 @@ int construct_vsag_search_param(uint8_t create_type, return ret; } -int construct_vsag_sindi_search_param(float query_prune_ratio, uint64_t n_candidate, +int construct_vsag_sindi_search_param(float query_prune_ratio, uint64_t n_candidate, char *result_param_str) { int ret = OB_SUCCESS; @@ -804,19 +798,19 @@ int construct_vsag_sindi_search_param(float query_prune_ratio, uint64_t n_candid int64_t pos = 0; int64_t buff_size = 0; int64_t buf_len = 1024; - if (OB_FAIL(databuff_printf(result_param_str, - buf_len, - pos, + if (OB_FAIL(databuff_printf(result_param_str, + buf_len, + pos, "{\"%s\":{", index_type_str))) { LOG_WARN("failed to fill result_param_str", K(ret), K(index_type_str)); - } else if (OB_FAIL(databuff_printf(result_param_str, - buf_len, - pos, + } else if (OB_FAIL(databuff_printf(result_param_str, + buf_len, + pos, "\"query_prune_ratio\":%f", query_prune_ratio))) { LOG_WARN("failed to fill result_param_str", K(ret), K(index_type_str)); - } else if (OB_FAIL(databuff_printf(result_param_str, - buf_len, - pos, + } else if (OB_FAIL(databuff_printf(result_param_str, + buf_len, + pos, ",\"n_candidate\":%lu}}", n_candidate))) { LOG_WARN("failed to fill result_param_str", K(ret), K(index_type_str)); } @@ -1159,7 +1153,7 @@ int knn_search(VectorIndexPtr &index_handler, float *query_vector, int dim, int64_t topk, const float *&dist, const int64_t *&ids, int64_t &result_size, int ef_search, bool need_extra_info, const char *&extra_infos, void *invalid, bool reverse_filter, - bool use_extra_info_filter, void *allocator, float valid_ratio, + bool use_extra_info_filter, void *allocator, float valid_ratio, float distance_threshold) { int ret = OB_SUCCESS; @@ -1341,7 +1335,7 @@ int fdeserialize(VectorIndexPtr &index_handler, ef_construction, ef_search, hnsw->get_allocator(), extra_info_size, refine_type, bq_bits_query, bq_use_fht, result_param_str))) { LOG_WARN("construct_vsag_create_param fail", K(ret), K(index_type)); - } + } } if (OB_FAIL(ret)) { } else { diff --git a/tools/windows/installer/wix_launch_configurator.wxs b/tools/windows/installer/wix_launch_configurator.wxs index 93ff8f3ce..f8692ff47 100644 --- a/tools/windows/installer/wix_launch_configurator.wxs +++ b/tools/windows/installer/wix_launch_configurator.wxs @@ -12,17 +12,6 @@ failed to link for some reason. --> - - - - - + - @@ -60,7 +42,7 @@ Only runs on full uninstall (not upgrades). --> + Condition="(REMOVE="ALL") AND (NOT UPGRADINGPRODUCTCODE)" /> @@ -70,10 +52,6 @@ Permanent="no" Part="last" Action="set" System="yes" /> - -