diff --git a/.bazelrc b/.bazelrc index 0171a70e20..a5dede7272 100644 --- a/.bazelrc +++ b/.bazelrc @@ -56,6 +56,8 @@ build --define=MEDIAPIPE_DISABLE_GPU=1 build --define=MEDIAPIPE_DISABLE=0 build --define=CLOUD_DISABLE=0 build --define=PYTHON_DISABLE=0 +build --define=GENAI_USE_BINARY=1 +build --define=USE_DROGON=1 # 0=legacy net_http, available only on linux # Sets the default Apple platform to macOS. build --apple_platform_type=macos @@ -91,7 +93,6 @@ build:linux --host_cxxopt=-std=c++17 build:linux --action_env PYTHON_BIN_PATH="/usr/bin/python3" build:linux --define PYTHON_BIN_PATH=/usr/bin/python3 -build:linux --define=USE_DROGON=1 # 0=legacy net_http, available only on linux # Security options ###################################################################################################### # TODO: Add equivalent windows flag build:linux --cxxopt=-fno-strict-overflow @@ -232,7 +233,6 @@ build:windows --deleted_packages=tensorflow/core/tfrt/stubs,tensorflow/compiler/ # Windows build ssl headers for GRPC workaround build:windows --override_repository="boringssl=C:\\opt\\boringSSL-SwiftPM" - #Add this parameter for windows local builds, its added on jenkins via windows_build.bat. Must be short path on C:\ for mediapipe to compile #startup --output_user_root=C:/b_tmp @@ -246,8 +246,6 @@ build:windows --action_env OpenVINO_DIR="c:/opt/openvino/runtime/cmake" build:windows --action_env OpenCV_DIR="c:/opt/opencv_4.12.0" build:windows --repo_env=HERMETIC_PYTHON_VERSION=3.12 -build:windows --define=USE_DROGON=1 -build:windows --define=GENAI_USE_BINARY=1 build:win_mp_on_py_on --config=windows build:win_mp_on_py_on --define=MEDIAPIPE_DISABLE=0 diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 013a0a27f4..8cc508d788 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -164,7 +164,7 @@ ARG ov_source_branch=c01cd93e24d1cd78bfbb401eed51c08fb93e0816 ARG ov_contrib_branch=master ARG ov_source_org=openvinotoolkit ARG ov_contrib_org=openvinotoolkit -ARG ov_use_binary=0 +ENV ov_use_binary=0 ARG DLDT_PACKAGE_URL ARG TEMP_DIR=/tmp/openvino_installer ARG CMAKE_BUILD_TYPE=Release @@ -199,9 +199,9 @@ WORKDIR $TEMP_DIR RUN if [ "$ov_use_binary" = "1" ] && [ "$DLDT_PACKAGE_URL" != "" ]; then true ; else exit 0 ; fi ; \ wget -nv $DLDT_PACKAGE_URL && \ mkdir /opt/intel && \ - tar -zxf ./*openvino_toolkit*.tgz -C /opt/intel && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino_2025 + tar -zxf ./*openvino_genai*.tar.gz -C /opt/intel && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino_2025 # install sample apps including benchmark_app # hadolint ignore=DL3041 @@ -214,17 +214,17 @@ RUN if [ -f /opt/intel/openvino/samples/cpp/build_samples.sh ]; then /opt/intel ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/:/opt/opencv/lib/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ -WORKDIR /openvino_tokenizers/ +WORKDIR /openvino_tokenizers/ ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 -ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +ARG ov_tokenizers_org=openvinotoolkit +ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" # hadolint ignore=DL3003 -RUN git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ - -WORKDIR /openvino_tokenizers/ -# Install the openvino_tokenizers python bindings and copy to OpenVINO location -RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/$ov_tokenizers_org/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + # Install the openvino_tokenizers python bindings and copy to OpenVINO location + if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp -r python/* /opt/intel/openvino/python/ ; \ cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ @@ -233,6 +233,16 @@ RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; \ fi +WORKDIR /openvino_genai/ +ARG ov_genai_branch=master +ARG ov_genai_org=openvinotoolkit +# hadolint ignore=DL3003 +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ + # Build OpenVINO Model Server WORKDIR /ovms COPY .bazelrc .user.bazelr[c] .bazelversion WORKSPACE /ovms/ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index b6c436fdb2..f70624492e 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -181,9 +181,9 @@ WORKDIR $TEMP_DIR RUN if [ "$ov_use_binary" == "1" ] && [ "$DLDT_PACKAGE_URL" != "" ]; then true ; else exit 0 ; fi ; \ wget -nv $DLDT_PACKAGE_URL && \ mkdir /opt/intel && \ - tar -zxf ./*openvino_toolkit*.tgz -C /opt/intel && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino_2025 + tar -zxf ./*openvino_genai*.tar.gz -C /opt/intel && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino_2025 WORKDIR / # install sample apps including benchmark_app @@ -193,16 +193,15 @@ RUN if [ -f /opt/intel/openvino/samples/cpp/build_samples.sh ]; then /opt/intel # OpenVINO Tokenizers extension ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so -WORKDIR /openvino_tokenizers/ +ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" ARG ov_tokenizers_branch=master -ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +ARG ov_tokenizers_org=openvinotoolkit # hadolint ignore=DL3003 -RUN git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive -RUN cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ - -# Install the openvino_tokenizers python bindings and copy to OpenVINO location -RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/$ov_tokenizers_org/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp -r python/* /opt/intel/openvino/python/ ; \ cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ @@ -211,6 +210,15 @@ RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; \ fi +ARG ov_genai_branch=master +ARG ov_genai_org=openvinotoolkit +# hadolint ignore=DL3003 +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ + # Add Nvidia dev tool if needed RUN apt-get update ; \ apt-get install -y --no-install-recommends opencl-clhpp-headers opencl-c-headers intel-opencl-icd && \ diff --git a/Makefile b/Makefile index c70fb37f4e..39370b8840 100644 --- a/Makefile +++ b/Makefile @@ -78,9 +78,12 @@ FUZZER_BUILD ?= 0 OV_SOURCE_BRANCH ?= 4a90bedcdd64dc4d6fee54cf03de6c5617b35a48 # master 2025/09/23 OV_CONTRIB_BRANCH ?= c39462ca8d7c550266dc70cdbfbe4fc8c5be0677 # master / 2024-10-31 OV_TOKENIZERS_BRANCH ?= bd47b33bcae913c59dcbe7e67ff52dbdf826ac32 # master 2025/09/18 +OV_GENAI_BRANCH ?= 845ba50e83e3f2f436d3794044a87f767f76bfd7 # master 2025/09/23 OV_SOURCE_ORG ?= openvinotoolkit OV_CONTRIB_ORG ?= openvinotoolkit +OV_GENAI_ORG ?= openvinotoolkit +OV_TOKENIZERS_ORG ?= openvinotoolkit TEST_LLM_PATH ?= "src/test/llm_testing" GPU_MODEL_PATH ?= "/tmp/face_detection_adas" @@ -137,6 +140,7 @@ ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) TARGET_DISTRO_PARAMS = " --//:distro=ubuntu" else ifeq ($(findstring redhat,$(BASE_OS)),redhat) TARGET_DISTRO_PARAMS = " --//:distro=redhat" + OV_USE_BINARY = 0 else $(error BASE_OS must be either ubuntu or redhat) endif @@ -162,11 +166,11 @@ ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) ifeq ($(BASE_OS_TAG),24.04) OS=ubuntu24 INSTALL_DRIVER_VERSION ?= "25.35.35096" - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.4.0-20052-4a90bedcdd6/openvino_toolkit_ubuntu24_2025.4.0.dev20250923_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20250923/openvino_genai_ubuntu24_2025.4.0.0.dev20250923_x86_64.tar.gz else ifeq ($(BASE_OS_TAG),22.04) OS=ubuntu22 INSTALL_DRIVER_VERSION ?= "24.39.31294" - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.4.0-20052-4a90bedcdd6/openvino_toolkit_ubuntu22_2025.4.0.dev20250923_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20250923/openvino_genai_ubuntu22_2025.4.0.0.dev20250923_x86_64.tar.gz endif endif ifeq ($(BASE_OS),redhat) @@ -175,7 +179,7 @@ ifeq ($(BASE_OS),redhat) BASE_IMAGE ?= registry.access.redhat.com/ubi9/ubi:$(BASE_OS_TAG_REDHAT) BASE_IMAGE_RELEASE=registry.access.redhat.com/ubi9/ubi-minimal:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.4.0-20052-4a90bedcdd6/openvino_toolkit_rhel8_2025.4.0.dev20250923_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20250923/openvino_genai_rhel8_2025.4.0.0.dev20250923_x86_64.tar.gz INSTALL_DRIVER_VERSION ?= "24.52.32224" endif @@ -212,7 +216,12 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg no_proxy=$(NO_PROXY)\ --build-arg ov_source_branch=$(OV_SOURCE_BRANCH)\ --build-arg ov_source_org=$(OV_SOURCE_ORG)\ + --build-arg ov_genai_org=$(OV_GENAI_ORG)\ + --build-arg ov_tokenizers_org=$(OV_TOKENIZERS_ORG)\ --build-arg ov_contrib_org=$(OV_CONTRIB_ORG)\ + --build-arg ov_contrib_branch=$(OV_CONTRIB_BRANCH)\ + --build-arg ov_tokenizers_branch=$(OV_TOKENIZERS_BRANCH)\ + --build-arg ov_genai_branch=$(OV_GENAI_BRANCH)\ --build-arg ov_use_binary=$(OV_USE_BINARY)\ --build-arg DLDT_PACKAGE_URL=$(DLDT_PACKAGE_URL)\ --build-arg CHECK_COVERAGE=$(CHECK_COVERAGE)\ @@ -226,8 +235,6 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg PROJECT_VERSION=$(PROJECT_VERSION)\ --build-arg BASE_IMAGE=$(BASE_IMAGE)\ --build-arg BASE_OS=$(BASE_OS)\ - --build-arg ov_contrib_branch=$(OV_CONTRIB_BRANCH)\ - --build-arg ov_tokenizers_branch=$(OV_TOKENIZERS_BRANCH)\ --build-arg INSTALL_RPMS_FROM_URL=$(INSTALL_RPMS_FROM_URL)\ --build-arg INSTALL_DRIVER_VERSION=$(INSTALL_DRIVER_VERSION)\ --build-arg RELEASE_BASE_IMAGE=$(BASE_IMAGE_RELEASE)\ diff --git a/WORKSPACE b/WORKSPACE index bfff7ea463..47d86e480f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -491,8 +491,8 @@ load("@ovms//third_party/aws-sdk-cpp:aws-sdk-cpp.bzl", "aws_sdk_cpp") aws_sdk_cpp() ### OpenVINO GenAI -load("@ovms//third_party/llm_engine:llm_engine.bzl", "llm_engine") -llm_engine() +### load("@ovms//third_party/llm_engine:llm_engine.bzl", "llm_engine") +###llm_engine() ### Libgit2 load("@ovms//third_party/libgit2:libgit2_engine.bzl", "libgit2_engine") diff --git a/create_package.sh b/create_package.sh index bc140b64ff..d18159af91 100755 --- a/create_package.sh +++ b/create_package.sh @@ -21,7 +21,8 @@ mkdir -vp /ovms_release/bin mkdir -vp /ovms_release/lib mkdir -vp /ovms_release/lib/custom_nodes -if [ -f /openvino_tokenizers/build/src/libopenvino_tokenizers.so ]; then cp -v /openvino_tokenizers/build/src/libopenvino_tokenizers.so /ovms_release/lib/ ; fi +# Do not link this tokenizer lib as it as old protobuf sentencepiece symbols the conflict with new protobuf from ovsm +if [ "$ov_use_binary" == "0" ] ; then cp -v /openvino_tokenizers/build/src/libopenvino_tokenizers.so /ovms_release/lib/ ; fi find /ovms/bazel-out/k8-*/bin -iname '*.so*' ! -type d ! -name "libgtest.so" ! -name "*params" ! -name "*.hana.*" ! -name "py_generate_pipeline.cpython*" ! -name "lib_node_*" ! -path "*test_python_binding*" ! -name "*libpython*" -exec cp -v {} /ovms_release/lib/ \; mv /ovms_release/lib/libcustom_node* /ovms_release/lib/custom_nodes/ diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index d6b81c0c06..50f0051f0b 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -1,3 +1,5 @@ +Dockerfile.redhat:228: thirdparty ==> third party, third-party +Dockerfile.ubuntu:206: thirdparty ==> third party, third-party client/common/resnet_labels.txt demos/common/python/classes.py demos/image_classification/go/labels.go diff --git a/src/BUILD b/src/BUILD index 2eaebc1137..cd647ac844 100644 --- a/src/BUILD +++ b/src/BUILD @@ -3159,10 +3159,8 @@ ovms_cc_library( "//src:libovms_ovinferrequestsqueue", "@mediapipe//mediapipe/framework:calculator_framework", "//third_party:openvino", - "@com_github_tencent_rapidjson//:rapidjson",] + select({ - "//conditions:default": ["//third_party:genai", "@llm_engine//:llm_engine"], - "//:not_genai_bin" : ["@llm_engine//:llm_engine"], - }), + "@com_github_tencent_rapidjson//:rapidjson", + "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, ) diff --git a/src/image_gen/BUILD b/src/image_gen/BUILD index 51815b8084..87056d4934 100644 --- a/src/image_gen/BUILD +++ b/src/image_gen/BUILD @@ -17,14 +17,6 @@ load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") load("//:common_settings.bzl", "ovms_cc_library") -ovms_cc_library( - name = "llm_engine", # in fact this is genai library, includes image gen, TODO: Change name? - srcs = [], - deps = ["@llm_engine//:llm_engine"], - visibility = ["//visibility:public"], - alwayslink = 1, -) - ovms_cc_library( name = "pipelines", hdrs = ["pipelines.hpp"], @@ -33,10 +25,7 @@ ovms_cc_library( "imagegenpipelineargs", "//src:libovmslogging", "//src:libovmsstring_utils", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, ) @@ -79,10 +68,7 @@ ovms_cc_library( "//src:image_conversion", "//src:libovmsstring_utils", "imagegenpipelineargs", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], ) @@ -97,10 +83,7 @@ ovms_cc_library( ":pipelines", "//src:image_conversion", ":imagegenutils", - ]+ select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, ) diff --git a/src/llm/BUILD b/src/llm/BUILD index 9e4ee702d6..9b69ae9268 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -17,14 +17,6 @@ load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") load("//:common_settings.bzl", "ovms_cc_library", "PYBIND_DEPS", "COPTS_PYTHON") -ovms_cc_library( - name = "llm_engine", - srcs = [], - deps = ["@llm_engine//:llm_engine"], - visibility = ["//visibility:public"], - additional_copts = COPTS_PYTHON -) - ovms_cc_library( name = "llmcalculator", srcs = ["http_llm_calculator.cc"], @@ -35,11 +27,8 @@ ovms_cc_library( "//src/kfserving_api:kfserving_api_cpp", "//src:libovmsprofiler", ":genai_servables", - "//src:httppayload" - ]+ select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//src:httppayload", + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON, alwayslink = 1, # needed, so the calculator can be registered by MediaPipe @@ -69,10 +58,7 @@ ovms_cc_library( "@stb//:image", ":openai_request", ":output_parsers", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -81,10 +67,7 @@ ovms_cc_library( name = "openai_request", hdrs = ["apis/openai_request.hpp"], srcs = [], - deps = select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + deps = ["//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -119,10 +102,7 @@ ovms_cc_library( "@com_github_tencent_rapidjson//:rapidjson", "//src:libovmslogging", "//src:libovmsstring_utils", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -141,10 +121,7 @@ ovms_cc_library( deps = [ ":openai_request", "//src:libovmslogging", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -191,10 +168,7 @@ ovms_cc_library( ":generation_config_builders", "//src:httppayload", "//src:libhttpclientconnection", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }) + select({ + "//third_party:genai",] + select({ "//:disable_python": [], "//:not_disable_python" : [":py_jinja_template_processor"], }), diff --git a/src/llm/servable.hpp b/src/llm/servable.hpp index a9528e0783..22c9483189 100644 --- a/src/llm/servable.hpp +++ b/src/llm/servable.hpp @@ -20,12 +20,11 @@ #include #include -#include "openvino/genai/text_streamer.hpp" - #pragma warning(push) -#pragma warning(disable : 4005 4309 6001 6385 6386 6326 6011 4005 4456 6246) +#pragma warning(disable : 4251 4005 4309 6001 6385 6386 6326 6011 4005 4456 6246) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "openvino/genai/text_streamer.hpp" #include "mediapipe/framework/calculator_graph.h" #pragma GCC diagnostic pop #pragma warning(pop) diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index 978742f5b2..736107de92 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -19,6 +19,7 @@ #include #include "../http_rest_api_handler.hpp" +#include "../servablemanagermodule.hpp" #include "../server.hpp" #include "rapidjson/document.h" #include "test_http_utils.hpp" @@ -399,9 +400,39 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values( "embeddings", "embeddings_ov")); +static bool isMpReady(const std::string name) { + ovms::Server& server = ovms::Server::instance(); + const ovms::Module* servableModule = server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); + if (!servableModule) { + return false; + } + ModelManager* manager = &dynamic_cast(servableModule)->getServableManager(); + auto mediapipeGraphDefinition = manager->getMediapipeFactory().findDefinitionByName(name); + if (!mediapipeGraphDefinition) { + return false; + } + return mediapipeGraphDefinition->getStatus().isAvailable(); +} + +static bool waitMpReady(const std::string name) { + SPDLOG_TRACE("waitMpReady:{}", name); + auto start = std::chrono::high_resolution_clock::now(); + while (!isMpReady(name) && + (std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - start).count() < SERVER_START_FROM_CONFIG_TIMEOUT_SECONDS / 4)) { + std::this_thread::sleep_for(std::chrono::microseconds(1000)); + } + + return isMpReady(name); +} + class EmbeddingsExtensionTest : public ::testing::Test { protected: static std::unique_ptr t; + static std::unique_ptr t1; + static std::unique_ptr t2; + static std::unique_ptr t3; + static std::unique_ptr t4; + static std::unique_ptr t5; public: std::unique_ptr handler; @@ -436,6 +467,38 @@ class EmbeddingsExtensionTest : public ::testing::Test { EXPECT_EQ(EXIT_SUCCESS, server.start(argc, argv)); })); EnsureServerStartedWithTimeout(server, 15); + + t1.reset(new std::thread([]() { + const std::string embeddings{"embeddings"}; + ASSERT_EQ(waitMpReady(embeddings), true); + })); + t2.reset(new std::thread([]() { + const std::string embeddings_no_norm{"embeddings_no_norm"}; + ASSERT_EQ(waitMpReady(embeddings_no_norm), true); + })); + t3.reset(new std::thread([]() { + const std::string embeddings_ov{"embeddings_ov"}; + ASSERT_EQ(waitMpReady(embeddings_ov), true); + })); + t4.reset(new std::thread([]() { + const std::string embeddings_ov_no_norm{"embeddings_ov_no_norm"}; + ASSERT_EQ(waitMpReady(embeddings_ov_no_norm), true); + })); + t5.reset(new std::thread([]() { + const std::string embeddings_ov_relative{"embeddings_ov_relative"}; + ASSERT_EQ(waitMpReady(embeddings_ov_relative), true); + })); + + if (t1->joinable()) + t1->join(); + if (t2->joinable()) + t2->join(); + if (t3->joinable()) + t3->join(); + if (t4->joinable()) + t4->join(); + if (t5->joinable()) + t5->join(); } void SetUp() { @@ -457,6 +520,16 @@ class EmbeddingsExtensionTest : public ::testing::Test { server.setShutdownRequest(1); t->join(); server.setShutdownRequest(0); + if (t1->joinable()) + t1->join(); + if (t2->joinable()) + t2->join(); + if (t3->joinable()) + t3->join(); + if (t4->joinable()) + t4->join(); + if (t5->joinable()) + t5->join(); } void TearDown() { @@ -467,6 +540,11 @@ class EmbeddingsExtensionTest : public ::testing::Test { } }; std::unique_ptr EmbeddingsExtensionTest::t; +std::unique_ptr EmbeddingsExtensionTest::t1; +std::unique_ptr EmbeddingsExtensionTest::t2; +std::unique_ptr EmbeddingsExtensionTest::t3; +std::unique_ptr EmbeddingsExtensionTest::t4; +std::unique_ptr EmbeddingsExtensionTest::t5; TEST_F(EmbeddingsExtensionTest, simplePositive) { std::string requestBody = R"( { diff --git a/third_party/genai/BUILD b/third_party/genai/BUILD index 82af5601fd..0fb74fc1ac 100644 --- a/third_party/genai/BUILD +++ b/third_party/genai/BUILD @@ -17,14 +17,22 @@ package( default_visibility = ["//visibility:public"], ) -#TODO + +cc_library( + name = "genai_headers", + hdrs = glob([ + "include/openvino/**/*.*" + ]), + strip_include_prefix = "include", + visibility = ["//visibility:public"], +) cc_library( name = "genai", srcs = glob([ - "" + "lib/intel64/libopenvino_genai.so", + "lib/intel64/libopenvino_genai.so.2540", ]), visibility = ["//visibility:public"], - deps = [ - ], + deps = [":genai_headers"], ) diff --git a/third_party/genai/genai_windows.BUILD b/third_party/genai/genai_windows.BUILD index ba3de7df7a..0f4cb24e09 100644 --- a/third_party/genai/genai_windows.BUILD +++ b/third_party/genai/genai_windows.BUILD @@ -21,7 +21,7 @@ package( cc_library( name = "genai_headers", hdrs = glob([ - "include/openvino/genai/*.*" + "include/openvino/**/*.*" ]), strip_include_prefix = "include", visibility = ["//visibility:public"], @@ -38,7 +38,7 @@ cc_import( cc_library( name = "genai", srcs = glob([ -# "bin\\intel64\\Release\\openvino_genai.dll", + "bin\\intel64\\Release\\openvino_genai.dll", "bin\\intel64\\Release\\icudt70.dll", "bin\\intel64\\Release\\icuuc70.dll", "bin\\intel64\\Release\\openvino_tokenizers.dll", @@ -46,7 +46,7 @@ cc_library( ]), visibility = ["//visibility:public"], deps = [ -# ":genai_lib", + ":genai_lib", ":genai_headers", ], ) diff --git a/third_party/llm_engine/BUILD b/third_party/llm_engine/BUILD deleted file mode 100644 index 9c4ccf9af2..0000000000 --- a/third_party/llm_engine/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tools/update_ov/update_ov.py b/tools/update_ov/update_ov.py index d23dbf8bf3..56c5538619 100644 --- a/tools/update_ov/update_ov.py +++ b/tools/update_ov/update_ov.py @@ -265,10 +265,10 @@ def update_openvino_genai(): # Update Makefile content with OpenVINO Tokenizers commit new_makefile_content = [] for line in makefile_content: - if "OV_TOKENIZERS_BRANCH ?=" in line: - print("Updating OV_TOKENIZERS_BRANCH") - indentation = line[:line.index("OV_TOKENIZERS_BRANCH ?=")] - new_line = f'{indentation}OV_TOKENIZERS_BRANCH ?= {tokenizers_commit} # master {tokenizers_commit_date}\n' + if "OV_GENAI_BRANCH ?=" in line: + print("Updating OV_GENAI_BRANCH") + indentation = line[:line.index("OV_GENAI_BRANCH ?=")] + new_line = f'{indentation}OV_GENAI_BRANCH ?= {tokenizers_commit} # master {tokenizers_commit_date}\n' print(new_line.lstrip()) new_makefile_content.append(new_line) else: diff --git a/windows_create_package.bat b/windows_create_package.bat index 8d9474ee62..171a329172 100644 --- a/windows_create_package.bat +++ b/windows_create_package.bat @@ -106,10 +106,15 @@ md %license_dest% if !errorlevel! neq 0 exit /b !errorlevel! copy C:\opt\opencv_4.12.0\etc\licenses\* %license_dest% if !errorlevel! neq 0 exit /b !errorlevel! -copy C:\%output_user_root%\openvino\docs\licensing\LICENSE %license_dest%openvino.LICENSE.txt -if !errorlevel! neq 0 exit /b !errorlevel! -copy C:\%output_user_root%\openvino\docs\licensing\LICENSE-GENAI %license_dest%LICENSE-GENAI.txt -if !errorlevel! neq 0 exit /b !errorlevel! +IF "%USE_OV_BINARY%"=="1" ( + copy C:\%output_user_root%\openvino\docs\licensing\LICENSE %license_dest%openvino.LICENSE.txt + if !errorlevel! neq 0 exit /b !errorlevel! + copy C:\%output_user_root%\openvino\docs\licensing\LICENSE-GENAI %license_dest%LICENSE-GENAI.txt + if !errorlevel! neq 0 exit /b !errorlevel! +) ELSE ( + copy C:\%output_user_root%\openvino\licenses %license_dest% + if !errorlevel! neq 0 exit /b !errorlevel! +) copy %cd%\release_files\LICENSE %cd%\dist\windows\ovms\ if !errorlevel! neq 0 exit /b !errorlevel! diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 76bc2e9de3..3f902faa73 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -129,12 +129,30 @@ IF /I EXIST %bash_path% ( echo [INFO] Msys installed in: %msys_path% ) +:: Set default USE_OV_BINARY if not set +if "%USE_OV_BINARY%"=="" ( + set "USE_OV_BINARY=1" +) + +set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" +set "genai_new_workspace=C:\\%output_user_root%\\openvino\\runtime" +:: Replace path to GenAi in ovms WORKSPACE file +if "!output_user_root!" neq "opt" ( + powershell -Command "(gc -Path WORKSPACE) -replace '%genai_workspace%', '%genai_new_workspace%' | Set-Content -Path WORKSPACE" + if !errorlevel! neq 0 exit /b !errorlevel! +) + +echo [INFO] USE_OV_BINARY=%USE_OV_BINARY% +IF "%USE_OV_BINARY%"=="0" ( + goto :install_openvino_from_src +) + ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::: Install in c:\PR-XXXX\ section started - once per build, reinstalled only with expunge clean :::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -::::::::::::::::::::::: GENAI/OPENVINO - reinstalled per build trigger +::::::::::::::::::::::: GENAI/OPENVINO install from ZIP - reinstalled per build trigger :: Set default GENAI_PACKAGE_URL if not set if "%GENAI_PACKAGE_URL%"=="" ( set "GENAI_PACKAGE_URL=https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20250923/openvino_genai_windows_2025.4.0.0.dev20250923_x86_64.zip" @@ -147,8 +165,6 @@ for %%F in ("%GENAI_PACKAGE_URL%") do set "genai_ver=%%~nxF" for %%F in ("%genai_ver%") do set "genai_dir=%%~nF" set "genai_zip=%BAZEL_SHORT_PATH%\%genai_ver%" -set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" -set "genai_new_workspace=C:\\%output_user_root%\\openvino\\runtime" echo [INFO] Installing GenAI: %genai_dir% ... :: Download GenAi @@ -184,15 +200,101 @@ IF /I EXIST %BAZEL_SHORT_PATH%\openvino ( mklink /d %BAZEL_SHORT_PATH%\openvino %BAZEL_SHORT_PATH%\%genai_dir% if !errorlevel! neq 0 exit /b !errorlevel! -:: Replace path to GenAi in ovms WORKSPACE file -if "!output_user_root!" neq "opt" ( - powershell -Command "(gc -Path WORKSPACE) -replace '%genai_workspace%', '%genai_new_workspace%' | Set-Content -Path WORKSPACE" - if !errorlevel! neq 0 exit /b !errorlevel! +echo [INFO] GenAI installed: %BAZEL_SHORT_PATH%\%genai_dir% +goto :finished_openvino + +:install_openvino_from_src +IF /I EXIST %BAZEL_SHORT_PATH%\openvino ( + rmdir /S /Q %BAZEL_SHORT_PATH%\openvino +) +if "%OV_SOURCE_BRANCH%"=="" ( + set "OV_SOURCE_BRANCH=4a90bedcdd64dc4d6fee54cf03de6c5617b35a48" +) +if "%OV_SOURCE_ORG%"=="" ( + set "OV_SOURCE_ORG=openvinotoolkit" +) +if "%TOKENIZER_SOURCE_ORG%"=="" ( + set "TOKENIZER_SOURCE_ORG=openvinotoolkit" +) +if "%TOKENIZER_SOURCE_BRANCH%"=="" ( + set "TOKENIZER_SOURCE_BRANCH=bd47b33bcae913c59dcbe7e67ff52dbdf826ac32" +) +if "%GENAI_SOURCE_ORG%"=="" ( + set "GENAI_SOURCE_ORG=openvinotoolkit" +) +if "%GENAI_SOURCE_BRANCH%"=="" ( + set "GENAI_SOURCE_BRANCH=845ba50e83e3f2f436d3794044a87f767f76bfd7" +) + +IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_src ( + git clone https://github.com/%OV_SOURCE_ORG%/openvino %BAZEL_SHORT_PATH%\openvino_src ) -:: Remove genai headers to be replaced by the ones from openvino_genai repository -rmdir /S /Q %BAZEL_SHORT_PATH%\%genai_dir%\runtime\include\openvino\genai -echo [INFO] GenAi installed: %BAZEL_SHORT_PATH%\%genai_dir% +set "BACK_CWD=%cd%" +cd %BAZEL_SHORT_PATH%\openvino_src +git fetch origin +git checkout %OV_SOURCE_BRANCH% +if !errorlevel! neq 0 exit /b !errorlevel! +git submodule update --init --recursive +if !errorlevel! neq 0 exit /b !errorlevel! +IF /I NOT EXIST build ( + mkdir build +) +cd build +set "TBB_DIR=" +cmake -G "Visual Studio 17 2022" -DENABLE_SAMPLES=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF .. +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --build . --config Release --verbose -j +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --install . --config Release --prefix %BAZEL_SHORT_PATH%\openvino +if !errorlevel! neq 0 exit /b !errorlevel! +call %BAZEL_SHORT_PATH%\openvino\setupvars.bat + +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +::::::::::::::::::::::: OpenVINO Tokenizers + +IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_tokenizers_src ( + git clone https://github.com/%TOKENIZER_SOURCE_ORG%/openvino_tokenizers.git %BAZEL_SHORT_PATH%\openvino_tokenizers_src +) +cd %BAZEL_SHORT_PATH%\openvino_tokenizers_src +git fetch origin +git checkout %TOKENIZER_SOURCE_BRANCH% +if !errorlevel! neq 0 exit /b !errorlevel! +IF /I NOT EXIST build ( + mkdir build +) +cd build +cmake -DCMAKE_BUILD_TYPE=Release .. +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --build . --config Release --verbose -j +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --install . --config Release --prefix %BAZEL_SHORT_PATH%\openvino +if !errorlevel! neq 0 exit /b !errorlevel! + +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +::::::::::::::::::::::: OpenVINO GenAI + +IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_genai_src ( + git clone https://github.com/%GENAI_SOURCE_ORG%/openvino.genai.git %BAZEL_SHORT_PATH%\openvino_genai_src +) +cd %BAZEL_SHORT_PATH%\openvino_genai_src +git fetch origin +git checkout %GENAI_SOURCE_BRANCH% +if !errorlevel! neq 0 exit /b !errorlevel! +IF /I NOT EXIST build ( + mkdir build +) +cd build +cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON .. +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --build . --config Release --verbose -j +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --install . --config Release --prefix %BAZEL_SHORT_PATH%\openvino +if !errorlevel! neq 0 exit /b !errorlevel! + +echo [INFO] OpenVINO from source installed: %BAZEL_SHORT_PATH%\openvino +cd !BACK_CWD! +:finished_openvino ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::: OpenCL headers echo [INFO] Installing OpenCL headers ...