diff --git a/.bazelrc b/.bazelrc index 0171a70e20..7beafd2e61 100644 --- a/.bazelrc +++ b/.bazelrc @@ -56,6 +56,7 @@ build --define=MEDIAPIPE_DISABLE_GPU=1 build --define=MEDIAPIPE_DISABLE=0 build --define=CLOUD_DISABLE=0 build --define=PYTHON_DISABLE=0 +build --define=USE_DROGON=1 # 0=legacy net_http, available only on linux # Sets the default Apple platform to macOS. build --apple_platform_type=macos @@ -91,7 +92,6 @@ build:linux --host_cxxopt=-std=c++17 build:linux --action_env PYTHON_BIN_PATH="/usr/bin/python3" build:linux --define PYTHON_BIN_PATH=/usr/bin/python3 -build:linux --define=USE_DROGON=1 # 0=legacy net_http, available only on linux # Security options ###################################################################################################### # TODO: Add equivalent windows flag build:linux --cxxopt=-fno-strict-overflow @@ -232,7 +232,6 @@ build:windows --deleted_packages=tensorflow/core/tfrt/stubs,tensorflow/compiler/ # Windows build ssl headers for GRPC workaround build:windows --override_repository="boringssl=C:\\opt\\boringSSL-SwiftPM" - #Add this parameter for windows local builds, its added on jenkins via windows_build.bat. Must be short path on C:\ for mediapipe to compile #startup --output_user_root=C:/b_tmp @@ -246,8 +245,6 @@ build:windows --action_env OpenVINO_DIR="c:/opt/openvino/runtime/cmake" build:windows --action_env OpenCV_DIR="c:/opt/opencv_4.12.0" build:windows --repo_env=HERMETIC_PYTHON_VERSION=3.12 -build:windows --define=USE_DROGON=1 -build:windows --define=GENAI_USE_BINARY=1 build:win_mp_on_py_on --config=windows build:win_mp_on_py_on --define=MEDIAPIPE_DISABLE=0 diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 013a0a27f4..9758df9320 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -161,9 +161,7 @@ RUN dnf install -y https://github.com/linux-test-project/lcov/releases/download ENV TF_SYSTEM_LIBS="curl" ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/test.log" ARG ov_source_branch=c01cd93e24d1cd78bfbb401eed51c08fb93e0816 -ARG ov_contrib_branch=master ARG ov_source_org=openvinotoolkit -ARG ov_contrib_org=openvinotoolkit ARG ov_use_binary=0 ARG DLDT_PACKAGE_URL ARG TEMP_DIR=/tmp/openvino_installer @@ -199,9 +197,9 @@ WORKDIR $TEMP_DIR RUN if [ "$ov_use_binary" = "1" ] && [ "$DLDT_PACKAGE_URL" != "" ]; then true ; else exit 0 ; fi ; \ wget -nv $DLDT_PACKAGE_URL && \ mkdir /opt/intel && \ - tar -zxf ./*openvino_toolkit*.tgz -C /opt/intel && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino_2025 + tar -zxf ./*openvino_genai*.tar.gz -C /opt/intel && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino_2025 # install sample apps including benchmark_app # hadolint ignore=DL3041 @@ -214,17 +212,17 @@ RUN if [ -f /opt/intel/openvino/samples/cpp/build_samples.sh ]; then /opt/intel ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/:/opt/opencv/lib/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ -WORKDIR /openvino_tokenizers/ +WORKDIR /openvino_tokenizers/ ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 -ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +ARG ov_tokenizers_org=openvinotoolkit +ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" # hadolint ignore=DL3003 -RUN git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ - -WORKDIR /openvino_tokenizers/ -# Install the openvino_tokenizers python bindings and copy to OpenVINO location -RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ +RUN git clone https://github.com/$ov_tokenizers_org/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + # Install the openvino_tokenizers python bindings and copy to OpenVINO location + if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp -r python/* /opt/intel/openvino/python/ ; \ cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ @@ -233,6 +231,17 @@ RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; \ fi +WORKDIR /openvino_genai/ +ARG ov_genai_branch=master +ARG ov_genai_org=openvinotoolkit +# hadolint ignore=DL3003 +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ ; \ + cp -r /openvino_genai/build/openvino_genai/*py* /opt/intel/openvino/python/ + # Build OpenVINO Model Server WORKDIR /ovms COPY .bazelrc .user.bazelr[c] .bazelversion WORKSPACE /ovms/ @@ -274,6 +283,12 @@ RUN if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; echo $'Metadata-Version: 1.0\nName: openvino\nVersion: 2025.4' > /opt/intel/openvino/python/openvino-4.dist-info/METADATA ENV PYTHONPATH=/opt/intel/openvino/python:/ovms/bazel-bin/src/python/binding +WORKDIR /patchelf +# hadolint ignore=DL3003 +RUN wget -q https://github.com/NixOS/patchelf/archive/0.10.tar.gz && \ + tar -xf 0.10.tar.gz && cd patchelf-0.10 && \ + ./bootstrap.sh && ./configure && make && make install + WORKDIR /ovms ARG PROJECT_VERSION="2025.4.0" @@ -321,6 +336,7 @@ FROM $BUILD_IMAGE as capi-build ARG CAPI_FLAGS="--strip=always --config=mp_off_py_off --//:distro=redhat" ARG JOBS=40 ARG LTO_ENABLE=OFF +WORKDIR /ovms RUN bazel build --jobs $JOBS ${CAPI_FLAGS} //src:ovms_shared # C api app with bazel @@ -338,12 +354,7 @@ RUN mkdir -p /ovms_release/lib/ ; find /ovms/bazel-out/k8-*/bin -iname 'libovms_ FROM $BUILD_IMAGE as pkg -RUN mkdir /patchelf -WORKDIR /patchelf -# hadolint ignore=DL3003 -RUN wget -q https://github.com/NixOS/patchelf/archive/0.10.tar.gz && \ - tar -xf 0.10.tar.gz && cd patchelf-0.10 && \ - ./bootstrap.sh && ./configure && make && make install +COPY --from=build /usr/local/bin/patchelf /usr/local/bin/patchelf WORKDIR / ARG BASE_OS=redhat diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index b23cfa30be..96bc7c4955 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -135,10 +135,8 @@ RUN if [ "$BASE_OS" == "ubuntu22" ] ; then python3 -m pip install "Jinja2==3.1.6 ARG JOBS=8 ARG ov_use_binary=0 ARG ov_source_org=openvinotoolkit -ARG ov_contrib_org=openvinotoolkit ARG DLDT_PACKAGE_URL ARG ov_source_branch=master -ARG ov_contrib_branch=master ARG CMAKE_BUILD_TYPE=Release ENV HDDL_INSTALL_DIR=/opt/intel/openvino/deployment_tools/inference_engine/external/hddl @@ -182,9 +180,9 @@ WORKDIR $TEMP_DIR RUN if [ "$ov_use_binary" == "1" ] && [ "$DLDT_PACKAGE_URL" != "" ]; then true ; else exit 0 ; fi ; \ wget -nv $DLDT_PACKAGE_URL && \ mkdir /opt/intel && \ - tar -zxf ./*openvino_toolkit*.tgz -C /opt/intel && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino_2025 + tar -zxf ./*openvino_genai*.tar.gz -C /opt/intel && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino_2025 WORKDIR / # install sample apps including benchmark_app @@ -194,16 +192,15 @@ RUN if [ -f /opt/intel/openvino/samples/cpp/build_samples.sh ]; then /opt/intel # OpenVINO Tokenizers extension ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so -WORKDIR /openvino_tokenizers/ +ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" ARG ov_tokenizers_branch=master -ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +ARG ov_tokenizers_org=openvinotoolkit # hadolint ignore=DL3003 -RUN git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive -RUN cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ - -# Install the openvino_tokenizers python bindings and copy to OpenVINO location -RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ +RUN git clone https://github.com/$ov_tokenizers_org/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp -r python/* /opt/intel/openvino/python/ ; \ cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ @@ -212,6 +209,16 @@ RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; \ fi +ARG ov_genai_branch=master +ARG ov_genai_org=openvinotoolkit +# hadolint ignore=DL3003 +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ ; \ + cp -r /openvino_genai/build/openvino_genai/*py* /opt/intel/openvino/python/ + # Add Nvidia dev tool if needed RUN apt-get update ; \ apt-get install -y --no-install-recommends opencl-clhpp-headers opencl-c-headers intel-opencl-icd && \ @@ -280,6 +287,13 @@ RUN bash -c "sed -i -e 's|REPLACE_PROJECT_VERSION|${PROJECT_VERSION}|g' /ovms/sr WORKDIR /usr/lib/x86_64-linux-gnu/ RUN ln -s libOpenCL.so.1 libOpenCL.so + +WORKDIR /patchelf +# hadolint ignore=DL3003 +RUN wget -q https://github.com/NixOS/patchelf/archive/0.10.tar.gz && \ + tar -xf 0.10.tar.gz && cd patchelf-0.10 && \ + ./bootstrap.sh && ./configure && make && make install + WORKDIR /ovms # Test Coverage COPY ci/check_coverage.bat /ovms/ @@ -309,7 +323,6 @@ COPY client /client RUN python3 -c "import json; m={'PROJECT_VERSION':'${PROJECT_VERSION}','OPENVINO backend':'$(/ovms/bazel-bin/src/ovms --version | grep backend | cut -d' ' -f3)', \ 'BAZEL_BUILD_FLAGS':'${debug_bazel_flags}${minitrace_flags}','BASE_IMAGE':'${BASE_IMAGE}' }; \ print(json.dumps(m,indent=4))" > /ovms/release_files/metadata.json - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # FROM $BUILD_IMAGE as capi-build @@ -319,6 +332,7 @@ SHELL ["/bin/bash", "-xo", "pipefail", "-c"] ARG CAPI_FLAGS="--strip=always --config mp_off_py_off --//:distro=ubuntu" ARG JOBS=40 ARG FUZZER_BUILD=0 +WORKDIR /ovms RUN bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:ovms_shared # C-API benchmark app @@ -340,12 +354,7 @@ RUN mkdir -p /ovms_release/lib/ ; find /ovms/bazel-out/k8-*/bin -iname 'libovms_ FROM build as pkg -RUN mkdir /patchelf -WORKDIR /patchelf -# hadolint ignore=DL3003 -RUN wget -q https://github.com/NixOS/patchelf/archive/0.10.tar.gz && \ - tar -xf 0.10.tar.gz && cd patchelf-0.10 && \ - ./bootstrap.sh && ./configure && make && make install +COPY --from=build /usr/local/bin/patchelf /usr/local/bin/patchelf WORKDIR / ARG BASE_OS=ubuntu diff --git a/Makefile b/Makefile index 91b1bb06f7..f001241ebb 100644 --- a/Makefile +++ b/Makefile @@ -75,15 +75,17 @@ FUZZER_BUILD ?= 0 # - uncomment source build section, comment binary section # - adjust binary version path - version variable is not passed to WORKSPACE file! -OV_SOURCE_BRANCH ?= 80e587bccec151a9acb0bc73fb96d007daa56e82 # master 2025/10/07 -OV_TOKENIZERS_BRANCH ?= 760f9140ab1fd330d62ec171673ca0705abe6aa0 # master 2025/10/07 +OV_SOURCE_BRANCH ?= ee21854d6821876e73fe8ec8f8eebd5fa1351927 # master 2025/10/14 +OV_TOKENIZERS_BRANCH ?= 760f9140ab1fd330d62ec171673ca0705abe6aa0 # master 2025/09/23 +OV_GENAI_BRANCH ?= 005f4d9fff7cfce6614d6b9d4d144273971080b4 # master 2025/10/14 OV_SOURCE_ORG ?= openvinotoolkit +OV_GENAI_ORG ?= openvinotoolkit +OV_TOKENIZERS_ORG ?= openvinotoolkit TEST_LLM_PATH ?= "src/test/llm_testing" GPU_MODEL_PATH ?= "/tmp/face_detection_adas" -OV_USE_BINARY ?= 1 APT_OV_PACKAGE ?= openvino-2022.1.0 # opt, dbg: BAZEL_BUILD_TYPE ?= opt @@ -133,8 +135,13 @@ endif ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) TARGET_DISTRO_PARAMS = " --//:distro=ubuntu" + OV_USE_BINARY ?= 1 else ifeq ($(findstring redhat,$(BASE_OS)),redhat) TARGET_DISTRO_PARAMS = " --//:distro=redhat" + OV_USE_BINARY ?= 0 + ifeq ($(OV_USE_BINARY),1) + $(error OV_USE_BINARY = 1 not supported on RHEL OS) + endif else $(error BASE_OS must be either ubuntu or redhat) endif @@ -160,11 +167,11 @@ ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) ifeq ($(BASE_OS_TAG),24.04) OS=ubuntu24 INSTALL_DRIVER_VERSION ?= "25.35.35096" - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.4.0-20120-80e587bccec/openvino_toolkit_ubuntu24_2025.4.0.dev20251007_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20251014/openvino_genai_ubuntu24_2025.4.0.0.dev20251014_x86_64.tar.gz else ifeq ($(BASE_OS_TAG),22.04) OS=ubuntu22 INSTALL_DRIVER_VERSION ?= "24.39.31294" - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.4.0-20120-80e587bccec/openvino_toolkit_ubuntu22_2025.4.0.dev20251007_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20251014/openvino_genai_ubuntu22_2025.4.0.0.dev20251014_x86_64.tar.gz endif endif ifeq ($(BASE_OS),redhat) @@ -173,7 +180,7 @@ ifeq ($(BASE_OS),redhat) BASE_IMAGE ?= registry.access.redhat.com/ubi9/ubi:$(BASE_OS_TAG_REDHAT) BASE_IMAGE_RELEASE=registry.access.redhat.com/ubi9/ubi-minimal:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.4.0-20120-80e587bccec/openvino_toolkit_rhel8_2025.4.0.dev20251007_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20251014/openvino_genai_rhel8_2025.4.0.0.dev20251014_x86_64.tar.gz INSTALL_DRIVER_VERSION ?= "24.52.32224" endif @@ -210,6 +217,10 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg no_proxy=$(NO_PROXY)\ --build-arg ov_source_branch=$(OV_SOURCE_BRANCH)\ --build-arg ov_source_org=$(OV_SOURCE_ORG)\ + --build-arg ov_genai_org=$(OV_GENAI_ORG)\ + --build-arg ov_tokenizers_org=$(OV_TOKENIZERS_ORG)\ + --build-arg ov_tokenizers_branch=$(OV_TOKENIZERS_BRANCH)\ + --build-arg ov_genai_branch=$(OV_GENAI_BRANCH)\ --build-arg ov_use_binary=$(OV_USE_BINARY)\ --build-arg DLDT_PACKAGE_URL=$(DLDT_PACKAGE_URL)\ --build-arg CHECK_COVERAGE=$(CHECK_COVERAGE)\ @@ -223,7 +234,6 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg PROJECT_VERSION=$(PROJECT_VERSION)\ --build-arg BASE_IMAGE=$(BASE_IMAGE)\ --build-arg BASE_OS=$(BASE_OS)\ - --build-arg ov_tokenizers_branch=$(OV_TOKENIZERS_BRANCH)\ --build-arg INSTALL_RPMS_FROM_URL=$(INSTALL_RPMS_FROM_URL)\ --build-arg INSTALL_DRIVER_VERSION=$(INSTALL_DRIVER_VERSION)\ --build-arg RELEASE_BASE_IMAGE=$(BASE_IMAGE_RELEASE)\ diff --git a/WORKSPACE b/WORKSPACE index bfff7ea463..a511e61d2e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -490,10 +490,6 @@ rules_pkg_dependencies() load("@ovms//third_party/aws-sdk-cpp:aws-sdk-cpp.bzl", "aws_sdk_cpp") aws_sdk_cpp() -### OpenVINO GenAI -load("@ovms//third_party/llm_engine:llm_engine.bzl", "llm_engine") -llm_engine() - ### Libgit2 load("@ovms//third_party/libgit2:libgit2_engine.bzl", "libgit2_engine") libgit2_engine() diff --git a/ci/build_test_OnCommit.groovy b/ci/build_test_OnCommit.groovy index 68eb18b3ce..9b93f2edfa 100644 --- a/ci/build_test_OnCommit.groovy +++ b/ci/build_test_OnCommit.groovy @@ -105,7 +105,7 @@ pipeline { sh "echo build --remote_cache=${env.OVMS_BAZEL_REMOTE_CACHE_URL} > .user.bazelrc" sh "echo test:linux --test_env https_proxy=${env.HTTPS_PROXY} >> .user.bazelrc" sh "echo test:linux --test_env http_proxy=${env.HTTP_PROXY} >> .user.bazelrc" - sh "make ovms_builder_image RUN_TESTS=0 OPTIMIZE_BUILDING_TESTS=1 OV_USE_BINARY=1 BASE_OS=redhat OVMS_CPP_IMAGE_TAG=${shortCommit} BUILD_IMAGE=openvino/model_server-build:${shortCommit}" + sh "make ovms_builder_image RUN_TESTS=0 OPTIMIZE_BUILDING_TESTS=1 OV_USE_BINARY=0 BASE_OS=redhat OVMS_CPP_IMAGE_TAG=${shortCommit} BUILD_IMAGE=openvino/model_server-build:${shortCommit}" } } stage('Build windows') { @@ -166,7 +166,7 @@ pipeline { label "${agent_name_linux}" } steps { - sh "make release_image RUN_TESTS=0 OV_USE_BINARY=1 BASE_OS=redhat OVMS_CPP_IMAGE_TAG=${shortCommit} BUILD_IMAGE=openvino/model_server-build:${shortCommit}" + sh "make release_image RUN_TESTS=0 OV_USE_BINARY=0 BASE_OS=redhat OVMS_CPP_IMAGE_TAG=${shortCommit} BUILD_IMAGE=openvino/model_server-build:${shortCommit}" sh "make run_lib_files_test BASE_OS=redhat OVMS_CPP_IMAGE_TAG=${shortCommit}" script { dir ('internal_tests'){ diff --git a/common_settings.bzl b/common_settings.bzl index 28803d6498..2a995d59c5 100644 --- a/common_settings.bzl +++ b/common_settings.bzl @@ -63,17 +63,6 @@ def create_config_settings(): name = "not_disable_mediapipe", negate = ":disable_mediapipe", ) - native.config_setting( - name = "genai_bin", - define_values = { - "GENAI_USE_BINARY": "1", - }, - visibility = ["//visibility:public"], - ) - more_selects.config_setting_negation( - name = "not_genai_bin", - negate = ":genai_bin", - ) native.config_setting( name = "enable_drogon", define_values = { diff --git a/create_package.sh b/create_package.sh index bc140b64ff..423b876c3e 100755 --- a/create_package.sh +++ b/create_package.sh @@ -15,13 +15,16 @@ # # This script should be used inside the build image to create a binary package based on the compiled artifacts - +set -e +# Set default value for variables +: "${FUZZER_BUILD:=0}" env mkdir -vp /ovms_release/bin mkdir -vp /ovms_release/lib mkdir -vp /ovms_release/lib/custom_nodes -if [ -f /openvino_tokenizers/build/src/libopenvino_tokenizers.so ]; then cp -v /openvino_tokenizers/build/src/libopenvino_tokenizers.so /ovms_release/lib/ ; fi +# Do not link this tokenizer lib as it has old protobuf sentencepiece symbols the conflict with new protobuf from ovsm +if [ "$ov_use_binary" == "0" ] ; then cp -v /openvino_tokenizers/build/src/libopenvino_tokenizers.so /ovms_release/lib/ ; fi find /ovms/bazel-out/k8-*/bin -iname '*.so*' ! -type d ! -name "libgtest.so" ! -name "*params" ! -name "*.hana.*" ! -name "py_generate_pipeline.cpython*" ! -name "lib_node_*" ! -path "*test_python_binding*" ! -name "*libpython*" -exec cp -v {} /ovms_release/lib/ \; mv /ovms_release/lib/libcustom_node* /ovms_release/lib/custom_nodes/ @@ -29,8 +32,8 @@ cd /ovms_release/lib/ ; rm -f libcurl.so* cd /ovms_release/lib/ ; rm -f libazurestorage.so.* ; ln -s libazurestorage.so libazurestorage.so.7 ;ln -s libazurestorage.so libazurestorage.so.7.5 cd /ovms_release/lib/ ; rm -f libcpprest.so.2.10 ; ln -s libcpprest.so libcpprest.so.2.10 -if [ -f /ovms_release/lib/libopenvino_genai.so ]; then cd /ovms_release/lib/ ; rm -f libopenvino_genai.so.* ; ln -s libopenvino_genai.so libopenvino_genai.so.2540 ; ln -s libopenvino_genai.so.2025.4.0.0 libopenvino_genai.so.2540 ; fi -if [ -f /ovms_release/lib/libopenvino_genai_c.so ]; then cd /ovms_release/lib/ ; rm -f libopenvino_genai_c.so* ; fi +if [ -f /ovms_release/lib/libopenvino_genai.so ]; then cd /ovms_release/lib/ ; rm -rf libopenvino_genai.so.* ; ln -s libopenvino_genai.so libopenvino_genai.so.2540 ; ln -s libopenvino_genai.so libopenvino_genai.so.2025.4.0.0 ; fi +if [ -e /ovms_release/lib/libopenvino_genai_c.so ]; then rm -rf /ovms_release/lib/libopenvino_genai_c.so* ; fi # Remove GPU plugin for CPU images? # Remove OpenCL for CPU images? @@ -69,8 +72,7 @@ if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then cp -r /opt/intel/openvino/pyt if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then mv /ovms_release/lib/pyovms.so /ovms_release/lib/python ; fi if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then echo $'#!/bin/bash\npython3 -m openvino_tokenizers.cli "$@"' > /ovms_release/bin/convert_tokenizer ; \ chmod +x /ovms_release/bin/convert_tokenizer ; fi -if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then cp -r /ovms/bazel-out/k8-opt/bin/external/llm_engine/libopenvino_genai/python/* /ovms_release/lib/python/ ; \ - mkdir -p /ovms_release/lib/python/openvino_genai-2025.4.dist-info ; \ +if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then mkdir -p /ovms_release/lib/python/openvino_genai-2025.4.dist-info ; \ echo $'Metadata-Version: 1.0\nName: openvino-genai\nVersion: 2025.4\nRequires-Python: >=3.9\nRequires-Dist: openvino-genai~=2025.4.0' > /ovms_release/lib/python/openvino_genai-2025.4.dist-info/METADATA; fi if [ -f /opt/intel/openvino/runtime/lib/intel64/plugins.xml ]; then cp /opt/intel/openvino/runtime/lib/intel64/plugins.xml /ovms_release/lib/ ; fi @@ -92,7 +94,7 @@ find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.so*' -exec cp -vP {} /ov patchelf --debug --set-rpath '$ORIGIN' /ovms_release/lib/libopenvino.so patchelf --debug --set-rpath '$ORIGIN' /ovms_release/lib/libopenvino_tokenizers.so patchelf --debug --set-rpath '$ORIGIN' /ovms_release/lib/lib*plugin.so -if [ -f /ovms_release/lib/libopenvino_nvidia_gpu_plugin.so ] && [ "$BASE_OS" != "redhat" ]; then patchelf --replace-needed libcutensor.so.1 /usr/lib/x86_64-linux-gnu/libcutensor/11/libcutensor.so.1 /ovms_release/lib/libopenvino_nvidia_gpu_plugin.so ; fi +if [ -f /ovms_release/lib/libopenvino_nvidia_gpu_plugin.so ] && [ "$BASE_OS" != "redhat" ]; then patchelf --replace-needed libcutensor.so.1 /usr/lib/x86_64-linux-gnu/libcutensor/11/libcutensor.so.1 /ovms_release/lib/libopenvino_nvidia_gpu_plugin.so ; fi cd /ovms cp -v /ovms/release_files/LICENSE /ovms_release/ @@ -103,7 +105,6 @@ if [ "$ov_use_binary" == "0" ] ; then cp -rf /openvino/LICENSE /ovms/release_fil mkdir -vp /ovms_release/include && cp /ovms/src/ovms.h /ovms_release/include ls -lahR /ovms_release/ - mkdir -p /ovms_pkg/${BASE_OS} cd /ovms_pkg/${BASE_OS} tar czf ovms.tar.gz --transform 's/ovms_release/ovms/' /ovms_release/ diff --git a/prepare_llm_models.sh b/prepare_llm_models.sh index 9909865707..9ab9956134 100755 --- a/prepare_llm_models.sh +++ b/prepare_llm_models.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # - +set -e if [ -z "$1" ]; then echo "Error: No directory specified." exit 1 @@ -75,7 +75,7 @@ else python3 demos/common/export_models/export_model.py text_generation --source_model "$CB_MODEL" --weight-format int8 --model_repository_path $1 fi if [ ! -f "$1/$CB_MODEL/$TOKENIZER_FILE" ]; then - echo "Models file $1/$CB_MODEL/$TOKENIZER_FILE does not exists." + echo "[ERROR] Models file $1/$CB_MODEL/$TOKENIZER_FILE does not exist." exit 1 fi @@ -85,7 +85,7 @@ else python3 demos/common/export_models/export_model.py text_generation --source_model "$VLM_MODEL" --weight-format int4 --kv_cache_precision u8 --model_repository_path $1 fi if [ ! -f "$1/$VLM_MODEL/$TOKENIZER_FILE" ]; then - echo "Model file $1/$VLM_MODEL/$TOKENIZER_FILE does not exists." + echo "[ERROR] Model file $1/$VLM_MODEL/$TOKENIZER_FILE does not exist." exit 1 fi @@ -95,7 +95,7 @@ else python3 demos/common/export_models/export_model.py embeddings --source_model "$EMBEDDING_MODEL" --weight-format int8 --model_repository_path $1 fi if [ ! -f "$1/$EMBEDDING_MODEL/embeddings/$LEGACY_MODEL_FILE" ]; then - echo "Models file $1/$EMBEDDING_MODEL/embeddings/$LEGACY_MODEL_FILE does not exists." + echo "[ERROR] Models file $1/$EMBEDDING_MODEL/embeddings/$LEGACY_MODEL_FILE does not exist." exit 1 fi @@ -105,7 +105,7 @@ else python3 demos/common/export_models/export_model.py embeddings_ov --source_model "$EMBEDDING_MODEL" --weight-format int8 --model_repository_path $1 --model_name $EMBEDDING_MODEL/ov fi if [ ! -f "$1/$EMBEDDING_MODEL/ov/$TOKENIZER_FILE" ]; then - echo "Model file "$1/$EMBEDDING_MODEL/ov/$TOKENIZER_FILE" does not exists." + echo "[ERROR] Model file "$1/$EMBEDDING_MODEL/ov/$TOKENIZER_FILE" does not exist." exit 1 fi @@ -115,7 +115,7 @@ else python3 demos/common/export_models/export_model.py rerank --source_model "$RERANK_MODEL" --weight-format int8 --model_repository_path $1 fi if [ ! -f "$1/$RERANK_MODEL/rerank/$LEGACY_MODEL_FILE" ]; then - echo "Model file $1/$RERANK_MODEL/rerank/$LEGACY_MODEL_FILE does not exists." + echo "[ERROR] Model file $1/$RERANK_MODEL/rerank/$LEGACY_MODEL_FILE does not exist." exit 1 fi @@ -125,7 +125,7 @@ else python3 demos/common/export_models/export_model.py rerank_ov --source_model "$RERANK_MODEL" --weight-format int8 --model_repository_path $1 --model_name $RERANK_MODEL/ov fi if [ ! -f "$1/$RERANK_MODEL/ov/$TOKENIZER_FILE" ]; then - echo "Model file $1/$RERANK_MODEL/ov/$TOKENIZER_FILE does not exists." + echo "[ERROR] Model file $1/$RERANK_MODEL/ov/$TOKENIZER_FILE does not exist." exit 1 fi @@ -136,7 +136,7 @@ else convert_tokenizer $QWEN3_MODEL --with_detokenizer -o $1/$QWEN3_MODEL fi if [ ! -f "$1/$QWEN3_MODEL/$TOKENIZER_FILE" ]; then - echo "Models file $1/$QWEN3_MODEL/$TOKENIZER_FILE does not exists." + echo "[ERROR] Models file $1/$QWEN3_MODEL/$TOKENIZER_FILE does not exist." exit 1 fi @@ -147,7 +147,7 @@ else convert_tokenizer $LLAMA3_MODEL --with_detokenizer -o $1/$LLAMA3_MODEL fi if [ ! -f "$1/$LLAMA3_MODEL/$TOKENIZER_FILE" ]; then - echo "Models file $1/$LLAMA3_MODEL/$TOKENIZER_FILE does not exists." + echo "[ERROR] Models file $1/$LLAMA3_MODEL/$TOKENIZER_FILE does not exist." exit 1 fi @@ -158,7 +158,7 @@ else convert_tokenizer $HERMES3_MODEL --with_detokenizer -o $1/$HERMES3_MODEL fi if [ ! -f "$1/$HERMES3_MODEL/$TOKENIZER_FILE" ]; then - echo "Models file $1/$HERMES3_MODEL/$TOKENIZER_FILE does not exists." + echo "[ERROR] Models file $1/$HERMES3_MODEL/$TOKENIZER_FILE does not exist." exit 1 fi @@ -169,7 +169,7 @@ else convert_tokenizer $PHI4_MODEL --with_detokenizer -o $1/$PHI4_MODEL fi if [ ! -f "$1/$PHI4_MODEL/$TOKENIZER_FILE" ]; then - echo "Models file $1/$PHI4_MODEL/$TOKENIZER_FILE does not exists." + echo "[ERROR] Models file $1/$PHI4_MODEL/$TOKENIZER_FILE does not exist." exit 1 fi @@ -180,7 +180,7 @@ else convert_tokenizer $MISTRAL_MODEL --with_detokenizer -o $1/$MISTRAL_MODEL fi if [ ! -f "$1/$MISTRAL_MODEL/$TOKENIZER_FILE" ]; then - echo "Models file $1/$MISTRAL_MODEL/$TOKENIZER_FILE does not exists." + echo "[ERROR] Models file $1/$MISTRAL_MODEL/$TOKENIZER_FILE does not exist." exit 1 fi @@ -191,6 +191,6 @@ else convert_tokenizer $GPT_OSS --with_detokenizer -o $1/$GPT_OSS fi if [ ! -f "$1/$GPT_OSS/$TOKENIZER_FILE" ]; then - echo "Models file $1/$GPT_OSS/$TOKENIZER_FILE does not exists." + echo "[ERROR] Models file $1/$GPT_OSS/$TOKENIZER_FILE does not exist." exit 1 fi diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index 62fc96c4ea..47664e85af 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -17,7 +17,6 @@ src/test/modelconfig_test.cpp:497: OptionA src/test/modelconfig_test.cpp:503: OptionA src/test/modelinstance_test.cpp:1093: THROUGHTPUT third_party/aws-sdk-cpp/aws-sdk-cpp.bz -third_party/llm_engine/llm_engine.bzl WORKSPACE:98: thirdparty demos/classification_using_paddlepaddle_model/python/utils/imagenet_class_index.json release_files/thirdparty-licenses/icu.LICENSE.txt:146: TaBE ==> table, tab diff --git a/src/BUILD b/src/BUILD index 2a8b60b5cd..3a99c20189 100644 --- a/src/BUILD +++ b/src/BUILD @@ -3155,10 +3155,8 @@ ovms_cc_library( "//src:libovms_ovinferrequestsqueue", "@mediapipe//mediapipe/framework:calculator_framework", "//third_party:openvino", - "@com_github_tencent_rapidjson//:rapidjson",] + select({ - "//conditions:default": ["//third_party:genai", "@llm_engine//:llm_engine"], - "//:not_genai_bin" : ["@llm_engine//:llm_engine"], - }), + "@com_github_tencent_rapidjson//:rapidjson", + "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, ) diff --git a/src/image_gen/BUILD b/src/image_gen/BUILD index 51815b8084..87056d4934 100644 --- a/src/image_gen/BUILD +++ b/src/image_gen/BUILD @@ -17,14 +17,6 @@ load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") load("//:common_settings.bzl", "ovms_cc_library") -ovms_cc_library( - name = "llm_engine", # in fact this is genai library, includes image gen, TODO: Change name? - srcs = [], - deps = ["@llm_engine//:llm_engine"], - visibility = ["//visibility:public"], - alwayslink = 1, -) - ovms_cc_library( name = "pipelines", hdrs = ["pipelines.hpp"], @@ -33,10 +25,7 @@ ovms_cc_library( "imagegenpipelineargs", "//src:libovmslogging", "//src:libovmsstring_utils", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, ) @@ -79,10 +68,7 @@ ovms_cc_library( "//src:image_conversion", "//src:libovmsstring_utils", "imagegenpipelineargs", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], ) @@ -97,10 +83,7 @@ ovms_cc_library( ":pipelines", "//src:image_conversion", ":imagegenutils", - ]+ select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, ) diff --git a/src/llm/BUILD b/src/llm/BUILD index 92ca4d9c70..404db1d9cd 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -17,14 +17,6 @@ load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") load("//:common_settings.bzl", "ovms_cc_library", "PYBIND_DEPS", "COPTS_PYTHON") -ovms_cc_library( - name = "llm_engine", - srcs = [], - deps = ["@llm_engine//:llm_engine"], - visibility = ["//visibility:public"], - additional_copts = COPTS_PYTHON -) - ovms_cc_library( name = "llmcalculator", srcs = ["http_llm_calculator.cc"], @@ -35,11 +27,8 @@ ovms_cc_library( "//src/kfserving_api:kfserving_api_cpp", "//src:libovmsprofiler", ":genai_servables", - "//src:httppayload" - ]+ select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//src:httppayload", + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON, alwayslink = 1, # needed, so the calculator can be registered by MediaPipe @@ -69,10 +58,7 @@ ovms_cc_library( "@stb//:image", ":openai_request", ":output_parsers", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -81,13 +67,9 @@ ovms_cc_library( name = "openai_request", hdrs = ["apis/openai_request.hpp"], srcs = [], - deps = [ - "//src/port:rapidjson_document", - ":apis_tool_schema_wrapper", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + deps = ["//third_party:genai", + "//src/port:rapidjson_document", + ":apis_tool_schema_wrapper",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -124,10 +106,8 @@ ovms_cc_library( deps = [ "@com_github_tencent_rapidjson//:rapidjson", ":io_processing_utils", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai", + ], ) ovms_cc_library( name = "io_processing_qwen3coder_tool_parser", @@ -140,10 +120,8 @@ ovms_cc_library( ":io_processing_utils", ":io_processing_base_output_parser", ":apis_tool_schema_wrapper", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai", + ], visibility = ["//visibility:public"], ) ovms_cc_library( # TODO split further so we don't have to recompile everything when changing one parser ... @@ -174,15 +152,13 @@ ovms_cc_library( # TODO split further so we don't have to recompile everything w "@com_github_tencent_rapidjson//:rapidjson", "//src:libovmslogging", "//src:libovmsstring_utils", + "//third_party:genai", ":partial_json_builder", ":io_processing_base_output_parser", ":io_processing_qwen3coder_tool_parser", ":io_processing_utils", ":apis_tool_schema_wrapper", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + ], visibility = ["//visibility:public"], ) @@ -200,10 +176,7 @@ ovms_cc_library( deps = [ ":openai_request", "//src:libovmslogging", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -250,10 +223,7 @@ ovms_cc_library( ":generation_config_builders", "//src:httppayload", "//src:libhttpclientconnection", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }) + select({ + "//third_party:genai",] + select({ "//:disable_python": [], "//:not_disable_python" : [":py_jinja_template_processor"], }), diff --git a/src/llm/io_processing/base_output_parser.hpp b/src/llm/io_processing/base_output_parser.hpp index 3a020eabf1..3afadfe756 100644 --- a/src/llm/io_processing/base_output_parser.hpp +++ b/src/llm/io_processing/base_output_parser.hpp @@ -18,10 +18,10 @@ #include #include #include -#include -#include #include #include +#include +#include #include #include diff --git a/src/llm/servable.hpp b/src/llm/servable.hpp index a9528e0783..22c9483189 100644 --- a/src/llm/servable.hpp +++ b/src/llm/servable.hpp @@ -20,12 +20,11 @@ #include #include -#include "openvino/genai/text_streamer.hpp" - #pragma warning(push) -#pragma warning(disable : 4005 4309 6001 6385 6386 6326 6011 4005 4456 6246) +#pragma warning(disable : 4251 4005 4309 6001 6385 6386 6326 6011 4005 4456 6246) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "openvino/genai/text_streamer.hpp" #include "mediapipe/framework/calculator_graph.h" #pragma GCC diagnostic pop #pragma warning(pop) diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index 978742f5b2..fba4de5858 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -19,6 +19,7 @@ #include #include "../http_rest_api_handler.hpp" +#include "../servablemanagermodule.hpp" #include "../server.hpp" #include "rapidjson/document.h" #include "test_http_utils.hpp" @@ -399,9 +400,35 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values( "embeddings", "embeddings_ov")); +static bool isMpReady(const std::string name) { + ovms::Server& server = ovms::Server::instance(); + const ovms::Module* servableModule = server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); + if (!servableModule) { + return false; + } + ModelManager* manager = &dynamic_cast(servableModule)->getServableManager(); + auto mediapipeGraphDefinition = manager->getMediapipeFactory().findDefinitionByName(name); + if (!mediapipeGraphDefinition) { + return false; + } + return mediapipeGraphDefinition->getStatus().isAvailable(); +} + +static bool waitMpReady(const std::string name) { + SPDLOG_TRACE("waitMpReady:{}", name); + auto start = std::chrono::high_resolution_clock::now(); + while (!isMpReady(name) && + (std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - start).count() < SERVER_START_FROM_CONFIG_TIMEOUT_SECONDS / 4)) { + std::this_thread::sleep_for(std::chrono::microseconds(1000)); + } + + return isMpReady(name); +} + class EmbeddingsExtensionTest : public ::testing::Test { protected: static std::unique_ptr t; + static std::unique_ptr t1; public: std::unique_ptr handler; @@ -436,6 +463,22 @@ class EmbeddingsExtensionTest : public ::testing::Test { EXPECT_EQ(EXIT_SUCCESS, server.start(argc, argv)); })); EnsureServerStartedWithTimeout(server, 15); + + t1.reset(new std::thread([]() { + const std::string embeddings{"embeddings"}; + ASSERT_EQ(waitMpReady(embeddings), true); + const std::string embeddings_no_norm{"embeddings_no_norm"}; + ASSERT_EQ(waitMpReady(embeddings_no_norm), true); + const std::string embeddings_ov{"embeddings_ov"}; + ASSERT_EQ(waitMpReady(embeddings_ov), true); + const std::string embeddings_ov_no_norm{"embeddings_ov_no_norm"}; + ASSERT_EQ(waitMpReady(embeddings_ov_no_norm), true); + const std::string embeddings_ov_relative{"embeddings_ov_relative"}; + ASSERT_EQ(waitMpReady(embeddings_ov_relative), true); + })); + + if (t1->joinable()) + t1->join(); } void SetUp() { @@ -457,6 +500,8 @@ class EmbeddingsExtensionTest : public ::testing::Test { server.setShutdownRequest(1); t->join(); server.setShutdownRequest(0); + if (t1->joinable()) + t1->join(); } void TearDown() { @@ -467,6 +512,7 @@ class EmbeddingsExtensionTest : public ::testing::Test { } }; std::unique_ptr EmbeddingsExtensionTest::t; +std::unique_ptr EmbeddingsExtensionTest::t1; TEST_F(EmbeddingsExtensionTest, simplePositive) { std::string requestBody = R"( { diff --git a/third_party/genai/BUILD b/third_party/genai/BUILD index 82af5601fd..5d719fe2fe 100644 --- a/third_party/genai/BUILD +++ b/third_party/genai/BUILD @@ -17,14 +17,21 @@ package( default_visibility = ["//visibility:public"], ) -#TODO + +cc_library( + name = "genai_headers", + hdrs = glob([ + "include/openvino/**/*.*" + ]), + strip_include_prefix = "include", + visibility = ["//visibility:public"], +) cc_library( name = "genai", srcs = glob([ - "" + "lib/intel64/libopenvino_genai.so", ]), visibility = ["//visibility:public"], - deps = [ - ], + deps = [":genai_headers"], ) diff --git a/third_party/genai/genai_windows.BUILD b/third_party/genai/genai_windows.BUILD index ba3de7df7a..0f4cb24e09 100644 --- a/third_party/genai/genai_windows.BUILD +++ b/third_party/genai/genai_windows.BUILD @@ -21,7 +21,7 @@ package( cc_library( name = "genai_headers", hdrs = glob([ - "include/openvino/genai/*.*" + "include/openvino/**/*.*" ]), strip_include_prefix = "include", visibility = ["//visibility:public"], @@ -38,7 +38,7 @@ cc_import( cc_library( name = "genai", srcs = glob([ -# "bin\\intel64\\Release\\openvino_genai.dll", + "bin\\intel64\\Release\\openvino_genai.dll", "bin\\intel64\\Release\\icudt70.dll", "bin\\intel64\\Release\\icuuc70.dll", "bin\\intel64\\Release\\openvino_tokenizers.dll", @@ -46,7 +46,7 @@ cc_library( ]), visibility = ["//visibility:public"], deps = [ -# ":genai_lib", + ":genai_lib", ":genai_headers", ], ) diff --git a/third_party/llm_engine/BUILD b/third_party/llm_engine/BUILD deleted file mode 100644 index 9c4ccf9af2..0000000000 --- a/third_party/llm_engine/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/tools/update_ov/update_ov.py b/tools/update_ov/update_ov.py index d23dbf8bf3..6dbe7ff00b 100644 --- a/tools/update_ov/update_ov.py +++ b/tools/update_ov/update_ov.py @@ -27,7 +27,6 @@ # and updates the Makefile with the new package links and commit SHAs. # - The script updates the requirements.txt file for the export models script with the new # OpenVINO and OpenVINO Tokenizers versions. -# - The script updates the llm_engine.bzl file with the new OpenVINO GenAI commit SHA. # - The script updates the windows_install_build_dependencies.bat file with the new OpenVINO GenAI package link. import requests @@ -209,24 +208,6 @@ def update_openvino_genai(): else: raise Exception("No commit link found.") - # Update OpenVINO GenAI commit in llm_engine.bzl - llm_engine_bzl_path = "third_party/llm_engine/llm_engine.bzl" - with open(llm_engine_bzl_path, 'r') as file: - llm_engine_bzl_content = file.readlines() - new_llm_engine_bzl_content = [] - for line in llm_engine_bzl_content: - if "commit =" in line: - print("Updating GenAI commit in llm_engine.bzl") - indentation = line[:line.index("commit =")] - new_line = f'{indentation}commit = "{commit}", # master {date_formatted}\n' - print(new_line.lstrip()) - new_llm_engine_bzl_content.append(new_line) - else: - new_llm_engine_bzl_content.append(line) - - with open(llm_engine_bzl_path, 'w') as file: - file.writelines(new_llm_engine_bzl_content) - # Fetch information about the tokenizers submodule tokenizers_commit = None tokenizers_commit_date = None diff --git a/windows_create_package.bat b/windows_create_package.bat index 8d9474ee62..4e941deff8 100644 --- a/windows_create_package.bat +++ b/windows_create_package.bat @@ -33,6 +33,11 @@ IF "%~2"=="--with_python" ( set "with_python=false" ) +:: Set default USE_OV_BINARY if not set +if "%USE_OV_BINARY%"=="" ( + set "USE_OV_BINARY=1" +) + if exist dist\windows\ovms ( rmdir /s /q dist\windows\ovms if !errorlevel! neq 0 exit /b !errorlevel! @@ -106,10 +111,15 @@ md %license_dest% if !errorlevel! neq 0 exit /b !errorlevel! copy C:\opt\opencv_4.12.0\etc\licenses\* %license_dest% if !errorlevel! neq 0 exit /b !errorlevel! -copy C:\%output_user_root%\openvino\docs\licensing\LICENSE %license_dest%openvino.LICENSE.txt -if !errorlevel! neq 0 exit /b !errorlevel! -copy C:\%output_user_root%\openvino\docs\licensing\LICENSE-GENAI %license_dest%LICENSE-GENAI.txt -if !errorlevel! neq 0 exit /b !errorlevel! +IF "%USE_OV_BINARY%"=="1" ( + copy C:\%output_user_root%\openvino\docs\licensing\LICENSE %license_dest%openvino.LICENSE.txt + if !errorlevel! neq 0 exit /b !errorlevel! + copy C:\%output_user_root%\openvino\docs\licensing\LICENSE-GENAI %license_dest%LICENSE-GENAI.txt + if !errorlevel! neq 0 exit /b !errorlevel! +) ELSE ( + copy C:\%output_user_root%\openvino\licenses %license_dest% + if !errorlevel! neq 0 exit /b !errorlevel! +) copy %cd%\release_files\LICENSE %cd%\dist\windows\ovms\ if !errorlevel! neq 0 exit /b !errorlevel! diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 105bc2cede..2922bc1d33 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -129,15 +129,33 @@ IF /I EXIST %bash_path% ( echo [INFO] Msys installed in: %msys_path% ) +:: Set default USE_OV_BINARY if not set +if "%USE_OV_BINARY%"=="" ( + set "USE_OV_BINARY=1" +) + +set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" +set "genai_new_workspace=C:\\%output_user_root%\\openvino\\runtime" +:: Replace path to GenAi in ovms WORKSPACE file +if "!output_user_root!" neq "opt" ( + powershell -Command "(gc -Path WORKSPACE) -replace '%genai_workspace%', '%genai_new_workspace%' | Set-Content -Path WORKSPACE" + if !errorlevel! neq 0 exit /b !errorlevel! +) + +echo [INFO] USE_OV_BINARY=%USE_OV_BINARY% +IF "%USE_OV_BINARY%"=="0" ( + goto :install_openvino_from_src +) + ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::: Install in c:\PR-XXXX\ section started - once per build, reinstalled only with expunge clean :::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -::::::::::::::::::::::: GENAI/OPENVINO - reinstalled per build trigger +::::::::::::::::::::::: GENAI/OPENVINO install from ZIP - reinstalled per build trigger :: Set default GENAI_PACKAGE_URL if not set if "%GENAI_PACKAGE_URL%"=="" ( - set "GENAI_PACKAGE_URL=https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20251007/openvino_genai_windows_2025.4.0.0.dev20251007_x86_64.zip" + set "GENAI_PACKAGE_URL=https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20251014/openvino_genai_windows_2025.4.0.0.dev20251014_x86_64.zip" ) :: Extract genai_ver from GENAI_PACKAGE_URL (filename) @@ -147,8 +165,6 @@ for %%F in ("%GENAI_PACKAGE_URL%") do set "genai_ver=%%~nxF" for %%F in ("%genai_ver%") do set "genai_dir=%%~nF" set "genai_zip=%BAZEL_SHORT_PATH%\%genai_ver%" -set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" -set "genai_new_workspace=C:\\%output_user_root%\\openvino\\runtime" echo [INFO] Installing GenAI: %genai_dir% ... :: Download GenAi @@ -184,15 +200,101 @@ IF /I EXIST %BAZEL_SHORT_PATH%\openvino ( mklink /d %BAZEL_SHORT_PATH%\openvino %BAZEL_SHORT_PATH%\%genai_dir% if !errorlevel! neq 0 exit /b !errorlevel! -:: Replace path to GenAi in ovms WORKSPACE file -if "!output_user_root!" neq "opt" ( - powershell -Command "(gc -Path WORKSPACE) -replace '%genai_workspace%', '%genai_new_workspace%' | Set-Content -Path WORKSPACE" - if !errorlevel! neq 0 exit /b !errorlevel! +echo [INFO] GenAI installed: %BAZEL_SHORT_PATH%\%genai_dir% +goto :finished_openvino + +:install_openvino_from_src +IF /I EXIST %BAZEL_SHORT_PATH%\openvino ( + rmdir /S /Q %BAZEL_SHORT_PATH%\openvino +) +if "%OV_SOURCE_BRANCH%"=="" ( + set "OV_SOURCE_BRANCH=ee21854d6821876e73fe8ec8f8eebd5fa1351927" +) +if "%OV_SOURCE_ORG%"=="" ( + set "OV_SOURCE_ORG=openvinotoolkit" +) +if "%TOKENIZER_SOURCE_ORG%"=="" ( + set "TOKENIZER_SOURCE_ORG=openvinotoolkit" +) +if "%TOKENIZER_SOURCE_BRANCH%"=="" ( + set "TOKENIZER_SOURCE_BRANCH=760f9140ab1fd330d62ec171673ca0705abe6aa0" +) +if "%GENAI_SOURCE_ORG%"=="" ( + set "GENAI_SOURCE_ORG=openvinotoolkit" +) +if "%GENAI_SOURCE_BRANCH%"=="" ( + set "GENAI_SOURCE_BRANCH=005f4d9fff7cfce6614d6b9d4d144273971080b4" +) + +IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_src ( + git clone https://github.com/%OV_SOURCE_ORG%/openvino %BAZEL_SHORT_PATH%\openvino_src ) -:: Remove genai headers to be replaced by the ones from openvino_genai repository -rmdir /S /Q %BAZEL_SHORT_PATH%\%genai_dir%\runtime\include\openvino\genai -echo [INFO] GenAi installed: %BAZEL_SHORT_PATH%\%genai_dir% +set "BACK_CWD=%cd%" +cd %BAZEL_SHORT_PATH%\openvino_src +git fetch origin +git checkout %OV_SOURCE_BRANCH% +if !errorlevel! neq 0 exit /b !errorlevel! +git submodule update --init --recursive +if !errorlevel! neq 0 exit /b !errorlevel! +IF /I NOT EXIST build ( + mkdir build +) +cd build +set "TBB_DIR=" +cmake -G "Visual Studio 17 2022" -DENABLE_SAMPLES=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF .. +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --build . --config Release --verbose -j +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --install . --config Release --prefix %BAZEL_SHORT_PATH%\openvino +if !errorlevel! neq 0 exit /b !errorlevel! +call %BAZEL_SHORT_PATH%\openvino\setupvars.bat + +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +::::::::::::::::::::::: OpenVINO Tokenizers + +IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_tokenizers_src ( + git clone https://github.com/%TOKENIZER_SOURCE_ORG%/openvino_tokenizers.git %BAZEL_SHORT_PATH%\openvino_tokenizers_src +) +cd %BAZEL_SHORT_PATH%\openvino_tokenizers_src +git fetch origin +git checkout %TOKENIZER_SOURCE_BRANCH% +if !errorlevel! neq 0 exit /b !errorlevel! +IF /I NOT EXIST build ( + mkdir build +) +cd build +cmake -DCMAKE_BUILD_TYPE=Release .. +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --build . --config Release --verbose -j +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --install . --config Release --prefix %BAZEL_SHORT_PATH%\openvino +if !errorlevel! neq 0 exit /b !errorlevel! + +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +::::::::::::::::::::::: OpenVINO GenAI + +IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_genai_src ( + git clone https://github.com/%GENAI_SOURCE_ORG%/openvino.genai.git %BAZEL_SHORT_PATH%\openvino_genai_src +) +cd %BAZEL_SHORT_PATH%\openvino_genai_src +git fetch origin +git checkout %GENAI_SOURCE_BRANCH% +if !errorlevel! neq 0 exit /b !errorlevel! +IF /I NOT EXIST build ( + mkdir build +) +cd build +cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON .. +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --build . --config Release --verbose -j +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --install . --config Release --prefix %BAZEL_SHORT_PATH%\openvino +if !errorlevel! neq 0 exit /b !errorlevel! + +echo [INFO] OpenVINO from source installed: %BAZEL_SHORT_PATH%\openvino +cd !BACK_CWD! +:finished_openvino ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::: OpenCL headers echo [INFO] Installing OpenCL headers ...