From edf7431aae3dddb9ab209111e3d65539e36ea6d3 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 17 Sep 2025 16:31:06 +0200 Subject: [PATCH 01/29] Linux compiles with binary --- .bazelrc | 8 +--- Dockerfile.redhat | 6 +-- Dockerfile.ubuntu | 18 ++++----- Makefile | 6 +-- WORKSPACE | 4 +- src/BUILD | 6 +-- src/image_gen/BUILD | 23 ++--------- src/llm/BUILD | 38 +++++-------------- src/llm/apis/openai_completions.hpp | 6 +-- src/llm/generation_config_header.hpp | 28 ++++++++++++++ .../base_generation_config_builder.cpp | 2 +- .../base_generation_config_builder.hpp | 2 +- src/llm/io_processing/base_output_parser.hpp | 2 +- .../generation_config_builder.hpp | 2 +- .../hermes3/generation_config_builder.cpp | 2 +- .../llama3/generation_config_builder.cpp | 2 +- .../phi4/generation_config_builder.cpp | 2 +- .../continuous_batching/llm_executor.hpp | 3 +- .../continuous_batching/servable.hpp | 3 +- .../servable_initializer.cpp | 3 +- .../language_model/legacy/legacy_executor.hpp | 3 +- src/llm/language_model/legacy/servable.hpp | 3 +- .../legacy/servable_initializer.cpp | 2 +- .../continuous_batching/servable.hpp | 2 - .../legacy/legacy_executor.hpp | 3 +- .../visual_language_model/legacy/servable.hpp | 3 +- .../legacy/servable_initializer.cpp | 2 +- src/test/llm/assisted_decoding_test.cpp | 2 +- src/test/llm/llmnode_test.cpp | 2 +- src/test/llm/llmtemplate_test.cpp | 2 +- third_party/genai/BUILD | 16 ++++++-- 31 files changed, 93 insertions(+), 113 deletions(-) create mode 100644 src/llm/generation_config_header.hpp diff --git a/.bazelrc b/.bazelrc index 0171a70e20..cbe0a96d08 100644 --- a/.bazelrc +++ b/.bazelrc @@ -56,6 +56,8 @@ build --define=MEDIAPIPE_DISABLE_GPU=1 build --define=MEDIAPIPE_DISABLE=0 build --define=CLOUD_DISABLE=0 build --define=PYTHON_DISABLE=0 +build --define=GENAI_USE_BINARY=1 +build --define=USE_DROGON=1 # 0=legacy net_http, available only on linux # Sets the default Apple platform to macOS. build --apple_platform_type=macos @@ -91,7 +93,6 @@ build:linux --host_cxxopt=-std=c++17 build:linux --action_env PYTHON_BIN_PATH="/usr/bin/python3" build:linux --define PYTHON_BIN_PATH=/usr/bin/python3 -build:linux --define=USE_DROGON=1 # 0=legacy net_http, available only on linux # Security options ###################################################################################################### # TODO: Add equivalent windows flag build:linux --cxxopt=-fno-strict-overflow @@ -232,7 +233,6 @@ build:windows --deleted_packages=tensorflow/core/tfrt/stubs,tensorflow/compiler/ # Windows build ssl headers for GRPC workaround build:windows --override_repository="boringssl=C:\\opt\\boringSSL-SwiftPM" - #Add this parameter for windows local builds, its added on jenkins via windows_build.bat. Must be short path on C:\ for mediapipe to compile #startup --output_user_root=C:/b_tmp @@ -246,8 +246,6 @@ build:windows --action_env OpenVINO_DIR="c:/opt/openvino/runtime/cmake" build:windows --action_env OpenCV_DIR="c:/opt/opencv_4.12.0" build:windows --repo_env=HERMETIC_PYTHON_VERSION=3.12 -build:windows --define=USE_DROGON=1 -build:windows --define=GENAI_USE_BINARY=1 build:win_mp_on_py_on --config=windows build:win_mp_on_py_on --define=MEDIAPIPE_DISABLE=0 @@ -262,8 +260,6 @@ test --experimental_ui_max_stdouterr_bytes=304857600 # envs below are required to succesfully run binaries in bazel as bazel is very strict in sandboxing test:linux --test_env LD_LIBRARY_PATH=/opt/opencv/lib/:/opt/intel/openvino/runtime/lib/intel64/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ -# genai hardcodes path during build time so without this so is not visible -test:linux --test_env OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so test:linux --test_env PYTHONPATH=/opt/intel/openvino/python:/ovms/bazel-bin/src/python/binding test:linux --test_env no_proxy=localhost diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 013a0a27f4..96627cd509 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -199,9 +199,9 @@ WORKDIR $TEMP_DIR RUN if [ "$ov_use_binary" = "1" ] && [ "$DLDT_PACKAGE_URL" != "" ]; then true ; else exit 0 ; fi ; \ wget -nv $DLDT_PACKAGE_URL && \ mkdir /opt/intel && \ - tar -zxf ./*openvino_toolkit*.tgz -C /opt/intel && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino_2025 + tar -zxf ./*openvino_genai*.tar.gz -C /opt/intel && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino_2025 # install sample apps including benchmark_app # hadolint ignore=DL3041 diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index b6c436fdb2..e9ba1af45f 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -181,9 +181,9 @@ WORKDIR $TEMP_DIR RUN if [ "$ov_use_binary" == "1" ] && [ "$DLDT_PACKAGE_URL" != "" ]; then true ; else exit 0 ; fi ; \ wget -nv $DLDT_PACKAGE_URL && \ mkdir /opt/intel && \ - tar -zxf ./*openvino_toolkit*.tgz -C /opt/intel && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino && \ - ln -s /opt/intel/*openvino_toolkit* /opt/intel/openvino_2025 + tar -zxf ./*openvino_genai*.tar.gz -C /opt/intel && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino && \ + ln -s /opt/intel/*openvino_genai* /opt/intel/openvino_2025 WORKDIR / # install sample apps including benchmark_app @@ -197,19 +197,15 @@ WORKDIR /openvino_tokenizers/ ARG ov_tokenizers_branch=master ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" -# hadolint ignore=DL3003 -RUN git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive -RUN cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ - -# Install the openvino_tokenizers python bindings and copy to OpenVINO location -RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ +RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp -r python/* /opt/intel/openvino/python/ ; \ cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ echo $'Metadata-Version: 1.0\nName: openvino-tokenizers\nVersion: 2025.4\nRequires-Python: >=3.9\nRequires-Dist: openvino~=2025.4.0' > /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info/METADATA ; \ - ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; \ - fi + ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; # Add Nvidia dev tool if needed RUN apt-get update ; \ diff --git a/Makefile b/Makefile index e7e0f548ee..c965e9b11c 100644 --- a/Makefile +++ b/Makefile @@ -162,11 +162,11 @@ ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) ifeq ($(BASE_OS_TAG),24.04) OS=ubuntu24 INSTALL_DRIVER_VERSION ?= "25.31.34666" - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.4.0-19947-05bf2133ac2/openvino_toolkit_ubuntu24_2025.4.0.dev20250911_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20250915/openvino_genai_ubuntu24_2025.4.0.0.dev20250915_x86_64.tar.gz else ifeq ($(BASE_OS_TAG),22.04) OS=ubuntu22 INSTALL_DRIVER_VERSION ?= "24.39.31294" - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.4.0-19947-05bf2133ac2/openvino_toolkit_ubuntu22_2025.4.0.dev20250911_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20250915/openvino_genai_ubuntu22_2025.4.0.0.dev20250915_x86_64.tar.gz endif endif ifeq ($(BASE_OS),redhat) @@ -175,7 +175,7 @@ ifeq ($(BASE_OS),redhat) BASE_IMAGE ?= registry.access.redhat.com/ubi9/ubi:$(BASE_OS_TAG_REDHAT) BASE_IMAGE_RELEASE=registry.access.redhat.com/ubi9/ubi-minimal:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.4.0-19947-05bf2133ac2/openvino_toolkit_rhel8_2025.4.0.dev20250911_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20250915/openvino_genai_rhel8_2025.4.0.0.dev20250915_x86_64.tar.gz INSTALL_DRIVER_VERSION ?= "24.52.32224" endif diff --git a/WORKSPACE b/WORKSPACE index bfff7ea463..47d86e480f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -491,8 +491,8 @@ load("@ovms//third_party/aws-sdk-cpp:aws-sdk-cpp.bzl", "aws_sdk_cpp") aws_sdk_cpp() ### OpenVINO GenAI -load("@ovms//third_party/llm_engine:llm_engine.bzl", "llm_engine") -llm_engine() +### load("@ovms//third_party/llm_engine:llm_engine.bzl", "llm_engine") +###llm_engine() ### Libgit2 load("@ovms//third_party/libgit2:libgit2_engine.bzl", "libgit2_engine") diff --git a/src/BUILD b/src/BUILD index 7509b069f9..a42190bac7 100644 --- a/src/BUILD +++ b/src/BUILD @@ -3159,10 +3159,8 @@ ovms_cc_library( "//src:libovms_ovinferrequestsqueue", "@mediapipe//mediapipe/framework:calculator_framework", "//third_party:openvino", - "@com_github_tencent_rapidjson//:rapidjson",] + select({ - "//conditions:default": ["//third_party:genai", "@llm_engine//:llm_engine"], - "//:not_genai_bin" : ["@llm_engine//:llm_engine"], - }), + "@com_github_tencent_rapidjson//:rapidjson", + "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, ) diff --git a/src/image_gen/BUILD b/src/image_gen/BUILD index 51815b8084..87056d4934 100644 --- a/src/image_gen/BUILD +++ b/src/image_gen/BUILD @@ -17,14 +17,6 @@ load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") load("//:common_settings.bzl", "ovms_cc_library") -ovms_cc_library( - name = "llm_engine", # in fact this is genai library, includes image gen, TODO: Change name? - srcs = [], - deps = ["@llm_engine//:llm_engine"], - visibility = ["//visibility:public"], - alwayslink = 1, -) - ovms_cc_library( name = "pipelines", hdrs = ["pipelines.hpp"], @@ -33,10 +25,7 @@ ovms_cc_library( "imagegenpipelineargs", "//src:libovmslogging", "//src:libovmsstring_utils", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, ) @@ -79,10 +68,7 @@ ovms_cc_library( "//src:image_conversion", "//src:libovmsstring_utils", "imagegenpipelineargs", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], ) @@ -97,10 +83,7 @@ ovms_cc_library( ":pipelines", "//src:image_conversion", ":imagegenutils", - ]+ select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, ) diff --git a/src/llm/BUILD b/src/llm/BUILD index 2c0898897c..dbd9fbf0a5 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -18,11 +18,11 @@ load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_prot load("//:common_settings.bzl", "ovms_cc_library", "PYBIND_DEPS", "COPTS_PYTHON") ovms_cc_library( - name = "llm_engine", + name = "genai_proxy", + hdrs = ["generation_config_header.hpp"], srcs = [], - deps = ["@llm_engine//:llm_engine"], + deps = ["//third_party:genai"], visibility = ["//visibility:public"], - additional_copts = COPTS_PYTHON ) ovms_cc_library( @@ -35,11 +35,8 @@ ovms_cc_library( "//src/kfserving_api:kfserving_api_cpp", "//src:libovmsprofiler", ":genai_servables", - "//src:httppayload" - ]+ select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + "//src:httppayload", + ":genai_proxy",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON, alwayslink = 1, # needed, so the calculator can be registered by MediaPipe @@ -69,10 +66,7 @@ ovms_cc_library( "@stb//:image", ":openai_request", ":output_parsers", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + ":genai_proxy",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -81,10 +75,7 @@ ovms_cc_library( name = "openai_request", hdrs = ["apis/openai_request.hpp"], srcs = [], - deps = select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + deps = [":genai_proxy",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -113,10 +104,7 @@ ovms_cc_library( "@com_github_tencent_rapidjson//:rapidjson", "//src:libovmslogging", "//src:libovmsstring_utils", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + ":genai_proxy",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -135,10 +123,7 @@ ovms_cc_library( deps = [ ":openai_request", "//src:libovmslogging", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }), + ":genai_proxy",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -185,10 +170,7 @@ ovms_cc_library( ":generation_config_builders", "//src:httppayload", "//src:libhttpclientconnection", - ] + select({ - "//conditions:default": ["//third_party:genai", ":llm_engine"], - "//:not_genai_bin" : [":llm_engine"], - }) + select({ + ":genai_proxy",] + select({ "//:disable_python": [], "//:not_disable_python" : [":py_jinja_template_processor"], }), diff --git a/src/llm/apis/openai_completions.hpp b/src/llm/apis/openai_completions.hpp index 55461d90c9..5dd9200f51 100644 --- a/src/llm/apis/openai_completions.hpp +++ b/src/llm/apis/openai_completions.hpp @@ -24,11 +24,8 @@ #include #include -#include -#include -#include #include -#include + #pragma warning(push) #pragma warning(disable : 6313) #include @@ -39,6 +36,7 @@ #include "absl/status/status.h" #pragma warning(pop) #include "../io_processing/output_parser.hpp" +#include "../generation_config_header.hpp" #include "openai_request.hpp" using namespace rapidjson; diff --git a/src/llm/generation_config_header.hpp b/src/llm/generation_config_header.hpp new file mode 100644 index 0000000000..c91cfc1fff --- /dev/null +++ b/src/llm/generation_config_header.hpp @@ -0,0 +1,28 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#pragma warning(push) +#pragma warning(disable : 4505) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wreorder" +#include +#include +#include +#include +#include +#include +#pragma GCC diagnostic pop +#pragma warning(pop) diff --git a/src/llm/io_processing/base_generation_config_builder.cpp b/src/llm/io_processing/base_generation_config_builder.cpp index 5ae2c55a85..f65572e94d 100644 --- a/src/llm/io_processing/base_generation_config_builder.cpp +++ b/src/llm/io_processing/base_generation_config_builder.cpp @@ -16,8 +16,8 @@ #include #include -#include #include "base_generation_config_builder.hpp" +#include "../generation_config_header.hpp" namespace ovms { void BaseGenerationConfigBuilder::setStructuralTagsConfig(const ov::genai::StructuralTagsConfig& structuralTagsConfig) { diff --git a/src/llm/io_processing/base_generation_config_builder.hpp b/src/llm/io_processing/base_generation_config_builder.hpp index 3af0f7c54d..733d769f72 100644 --- a/src/llm/io_processing/base_generation_config_builder.hpp +++ b/src/llm/io_processing/base_generation_config_builder.hpp @@ -16,9 +16,9 @@ #pragma once #include -#include #include #include "../apis/openai_request.hpp" +#include "../generation_config_header.hpp" namespace ovms { diff --git a/src/llm/io_processing/base_output_parser.hpp b/src/llm/io_processing/base_output_parser.hpp index d8ff2f0745..888c941d07 100644 --- a/src/llm/io_processing/base_output_parser.hpp +++ b/src/llm/io_processing/base_output_parser.hpp @@ -16,7 +16,6 @@ #pragma once #include -#include #include #include #include @@ -29,6 +28,7 @@ #include #pragma warning(pop) +#include "../generation_config_header.hpp" #include "partial_json_builder.hpp" namespace ovms { diff --git a/src/llm/io_processing/generation_config_builder.hpp b/src/llm/io_processing/generation_config_builder.hpp index 9dd3289596..5d45c5c279 100644 --- a/src/llm/io_processing/generation_config_builder.hpp +++ b/src/llm/io_processing/generation_config_builder.hpp @@ -18,9 +18,9 @@ #include #include #include -#include #include #include "base_generation_config_builder.hpp" +#include "../generation_config_header.hpp" #include "phi4/generation_config_builder.hpp" #include "llama3/generation_config_builder.hpp" #include "hermes3/generation_config_builder.hpp" diff --git a/src/llm/io_processing/hermes3/generation_config_builder.cpp b/src/llm/io_processing/hermes3/generation_config_builder.cpp index c42b876d83..2ea50d6b51 100644 --- a/src/llm/io_processing/hermes3/generation_config_builder.cpp +++ b/src/llm/io_processing/hermes3/generation_config_builder.cpp @@ -16,8 +16,8 @@ #include #include -#include +#include "../../generation_config_header.hpp" #include "generation_config_builder.hpp" namespace ovms { diff --git a/src/llm/io_processing/llama3/generation_config_builder.cpp b/src/llm/io_processing/llama3/generation_config_builder.cpp index fa8b7f4058..d9f17c6301 100644 --- a/src/llm/io_processing/llama3/generation_config_builder.cpp +++ b/src/llm/io_processing/llama3/generation_config_builder.cpp @@ -16,9 +16,9 @@ #include #include -#include #include "generation_config_builder.hpp" +#include "../../generation_config_header.hpp" namespace ovms { diff --git a/src/llm/io_processing/phi4/generation_config_builder.cpp b/src/llm/io_processing/phi4/generation_config_builder.cpp index 1142cfc4c9..0df7e9f4ac 100644 --- a/src/llm/io_processing/phi4/generation_config_builder.cpp +++ b/src/llm/io_processing/phi4/generation_config_builder.cpp @@ -16,9 +16,9 @@ #include #include -#include #include "generation_config_builder.hpp" +#include "../../generation_config_header.hpp" namespace ovms { diff --git a/src/llm/language_model/continuous_batching/llm_executor.hpp b/src/llm/language_model/continuous_batching/llm_executor.hpp index f396d78dc3..11dc00174f 100644 --- a/src/llm/language_model/continuous_batching/llm_executor.hpp +++ b/src/llm/language_model/continuous_batching/llm_executor.hpp @@ -24,8 +24,7 @@ #include #include -#include - +#include "../../generation_config_header.hpp" #include "../../../logging.hpp" #include "../../../profiler.hpp" diff --git a/src/llm/language_model/continuous_batching/servable.hpp b/src/llm/language_model/continuous_batching/servable.hpp index def574e009..23711daa1b 100644 --- a/src/llm/language_model/continuous_batching/servable.hpp +++ b/src/llm/language_model/continuous_batching/servable.hpp @@ -17,8 +17,7 @@ #include #include -#include - +#include "../../generation_config_header.hpp" #include "../../servable.hpp" #include "src/llm/llm_calculator.pb.h" diff --git a/src/llm/language_model/continuous_batching/servable_initializer.cpp b/src/llm/language_model/continuous_batching/servable_initializer.cpp index ec0d011dd4..ac01198803 100644 --- a/src/llm/language_model/continuous_batching/servable_initializer.cpp +++ b/src/llm/language_model/continuous_batching/servable_initializer.cpp @@ -18,8 +18,6 @@ #include #include #include -#include -#include #include #include @@ -31,6 +29,7 @@ #pragma GCC diagnostic pop #pragma warning(pop) +#include "../../generation_config_header.hpp" #include "../../../json_parser.hpp" #include "../../../logging.hpp" #include "../../../mediapipe_internal/mediapipe_utils.hpp" diff --git a/src/llm/language_model/legacy/legacy_executor.hpp b/src/llm/language_model/legacy/legacy_executor.hpp index 5bb3365bd2..83ac51d41a 100644 --- a/src/llm/language_model/legacy/legacy_executor.hpp +++ b/src/llm/language_model/legacy/legacy_executor.hpp @@ -25,8 +25,7 @@ #include #include -#include "openvino/genai/llm_pipeline.hpp" - +#include "../../generation_config_header.hpp" #include "../../../logging.hpp" #include "../../../profiler.hpp" diff --git a/src/llm/language_model/legacy/servable.hpp b/src/llm/language_model/legacy/servable.hpp index b2f0aa01d1..820e21baa2 100644 --- a/src/llm/language_model/legacy/servable.hpp +++ b/src/llm/language_model/legacy/servable.hpp @@ -17,8 +17,7 @@ #include #include -#include "openvino/genai/llm_pipeline.hpp" - +#include "../../generation_config_header.hpp" #include "../../servable.hpp" #include "legacy_executor.hpp" #include "src/llm/llm_calculator.pb.h" diff --git a/src/llm/language_model/legacy/servable_initializer.cpp b/src/llm/language_model/legacy/servable_initializer.cpp index 4ee7d4820a..f1012b9d45 100644 --- a/src/llm/language_model/legacy/servable_initializer.cpp +++ b/src/llm/language_model/legacy/servable_initializer.cpp @@ -18,7 +18,6 @@ #include #include -#include "openvino/genai/llm_pipeline.hpp" #include #include @@ -30,6 +29,7 @@ #pragma GCC diagnostic pop #pragma warning(pop) +#include "../../generation_config_header.hpp" #include "../../../json_parser.hpp" #include "../../../logging.hpp" #include "../../../mediapipe_internal/mediapipe_utils.hpp" diff --git a/src/llm/visual_language_model/continuous_batching/servable.hpp b/src/llm/visual_language_model/continuous_batching/servable.hpp index 88c78e3dcb..7f468a9906 100644 --- a/src/llm/visual_language_model/continuous_batching/servable.hpp +++ b/src/llm/visual_language_model/continuous_batching/servable.hpp @@ -18,8 +18,6 @@ #include #include -#include - #include "../../language_model/continuous_batching/servable.hpp" namespace ovms { diff --git a/src/llm/visual_language_model/legacy/legacy_executor.hpp b/src/llm/visual_language_model/legacy/legacy_executor.hpp index b2e33ab7cf..013ef10cdd 100644 --- a/src/llm/visual_language_model/legacy/legacy_executor.hpp +++ b/src/llm/visual_language_model/legacy/legacy_executor.hpp @@ -25,8 +25,7 @@ #include #include -#include - +#include "../../generation_config_header.hpp" #include "../../../logging.hpp" #include "../../../profiler.hpp" diff --git a/src/llm/visual_language_model/legacy/servable.hpp b/src/llm/visual_language_model/legacy/servable.hpp index 0e77be8010..cee3d803f9 100644 --- a/src/llm/visual_language_model/legacy/servable.hpp +++ b/src/llm/visual_language_model/legacy/servable.hpp @@ -18,8 +18,7 @@ #include #include -#include - +#include "../../generation_config_header.hpp" #include "../../servable.hpp" #include "legacy_executor.hpp" #include "src/llm/llm_calculator.pb.h" diff --git a/src/llm/visual_language_model/legacy/servable_initializer.cpp b/src/llm/visual_language_model/legacy/servable_initializer.cpp index 5fddcbc98e..7d809343ca 100644 --- a/src/llm/visual_language_model/legacy/servable_initializer.cpp +++ b/src/llm/visual_language_model/legacy/servable_initializer.cpp @@ -18,7 +18,6 @@ #include #include -#include #include #include @@ -34,6 +33,7 @@ #include "../../../logging.hpp" #include "../../../mediapipe_internal/mediapipe_utils.hpp" #include "../../../status.hpp" +#include "../../generation_config_header.hpp" #include "servable.hpp" #include "servable_initializer.hpp" diff --git a/src/test/llm/assisted_decoding_test.cpp b/src/test/llm/assisted_decoding_test.cpp index 86f4f26e57..2db1ed6a80 100644 --- a/src/test/llm/assisted_decoding_test.cpp +++ b/src/test/llm/assisted_decoding_test.cpp @@ -24,7 +24,6 @@ #include #include -#include #include #if (PYTHON_DISABLE == 0) #pragma warning(push) @@ -37,6 +36,7 @@ #include "../../http_status_code.hpp" #include "../../json_parser.hpp" #include "../../llm/apis/openai_completions.hpp" +#include "../../llm/generation_config_header.hpp" #include "../../llm/language_model/continuous_batching/servable.hpp" #include "../../llm/language_model/continuous_batching/llm_executor.hpp" #include "../../llm/text_utils.hpp" diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 1a96fabb99..4cf1b0a7d1 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -24,7 +24,6 @@ #include #include -#include #include #if (PYTHON_DISABLE == 0) #pragma warning(push) @@ -37,6 +36,7 @@ #include "../../http_status_code.hpp" #include "../../json_parser.hpp" #include "../../llm/apis/openai_completions.hpp" +#include "../../llm/generation_config_header.hpp" #include "../../llm/language_model/continuous_batching/llm_executor.hpp" #include "../../llm/language_model/continuous_batching/servable.hpp" #include "../../llm/servable.hpp" diff --git a/src/test/llm/llmtemplate_test.cpp b/src/test/llm/llmtemplate_test.cpp index 848d795a05..2f01b73050 100644 --- a/src/test/llm/llmtemplate_test.cpp +++ b/src/test/llm/llmtemplate_test.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #pragma warning(push) #pragma warning(disable : 6326 28182 6011 28020) @@ -32,6 +31,7 @@ #include "../../http_payload.hpp" #include "../../http_rest_api_handler.hpp" #include "../../httpservermodule.hpp" +#include "../../llm/generation_config_header.hpp" #include "../../llm/language_model/continuous_batching/servable.hpp" #include "../../llm/language_model/continuous_batching/servable_initializer.hpp" #include "../../llm/py_jinja_template_processor.hpp" diff --git a/third_party/genai/BUILD b/third_party/genai/BUILD index 82af5601fd..4dbfe81c7a 100644 --- a/third_party/genai/BUILD +++ b/third_party/genai/BUILD @@ -17,14 +17,22 @@ package( default_visibility = ["//visibility:public"], ) -#TODO + +cc_library( + name = "genai_headers", + hdrs = glob([ + "include/openvino/**/*.*" + ]), + strip_include_prefix = "include", + visibility = ["//visibility:public"], +) cc_library( name = "genai", srcs = glob([ - "" + "lib/intel64/libopenvino_genai.so", + "lib/intel64/libopenvino_tokenizers.so", ]), visibility = ["//visibility:public"], - deps = [ - ], + deps = [":genai_headers"], ) From 664ee824bbb087893f160b540f0e47ebc56819f4 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 17 Sep 2025 16:36:44 +0200 Subject: [PATCH 02/29] Fix redhat --- Dockerfile.redhat | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 96627cd509..bc5da398c6 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -218,20 +218,15 @@ WORKDIR /openvino_tokenizers/ ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" -# hadolint ignore=DL3003 -RUN git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive -RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ - -WORKDIR /openvino_tokenizers/ -# Install the openvino_tokenizers python bindings and copy to OpenVINO location -RUN if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ +RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS} ${LTO_CXX_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp -r python/* /opt/intel/openvino/python/ ; \ cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ echo $'Metadata-Version: 1.0\nName: openvino-tokenizers\nVersion: 2025.4\nRequires-Python: >=3.9\nRequires-Dist: openvino~=2025.4.0' > /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info/METADATA ; \ - ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; \ - fi + ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; # Build OpenVINO Model Server WORKDIR /ovms From db69ba4edf82d52f104480528dd42d7169761ae7 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 17 Sep 2025 16:39:33 +0200 Subject: [PATCH 03/29] Shared flag --- Dockerfile.redhat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index bc5da398c6..f662fdfa91 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -220,7 +220,7 @@ ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ - cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS} ${LTO_CXX_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp -r python/* /opt/intel/openvino/python/ ; \ cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ From 7af85c2b1da4fb55ec45bd86d321eb8938d97b42 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 18 Sep 2025 09:06:58 +0200 Subject: [PATCH 04/29] fix sdl --- Dockerfile.redhat | 1 + Dockerfile.ubuntu | 1 + 2 files changed, 2 insertions(+) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index f662fdfa91..551f64df8f 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -217,6 +217,7 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/: WORKDIR /openvino_tokenizers/ ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 +# hadolint ignore=DL3003 ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index e9ba1af45f..b0d2aa1c85 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -197,6 +197,7 @@ WORKDIR /openvino_tokenizers/ ARG ov_tokenizers_branch=master ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +# hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ From 3945a971a817d51920805981239a0e7d1ae9565a Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 18 Sep 2025 09:28:05 +0200 Subject: [PATCH 05/29] Fix sdl --- Dockerfile.redhat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 551f64df8f..9ee92803a7 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -217,8 +217,8 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/: WORKDIR /openvino_tokenizers/ ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 -# hadolint ignore=DL3003 ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +# hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ From 0a298768b46cc22b7d8b9b7425cc2f99a1770ef1 Mon Sep 17 00:00:00 2001 From: rasapala Date: Thu, 18 Sep 2025 10:45:27 +0200 Subject: [PATCH 06/29] Fix windows build --- src/llm/servable.hpp | 5 ++--- third_party/genai/genai_windows.BUILD | 6 +++--- windows_install_build_dependencies.bat | 3 +-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/llm/servable.hpp b/src/llm/servable.hpp index a9528e0783..22c9483189 100644 --- a/src/llm/servable.hpp +++ b/src/llm/servable.hpp @@ -20,12 +20,11 @@ #include #include -#include "openvino/genai/text_streamer.hpp" - #pragma warning(push) -#pragma warning(disable : 4005 4309 6001 6385 6386 6326 6011 4005 4456 6246) +#pragma warning(disable : 4251 4005 4309 6001 6385 6386 6326 6011 4005 4456 6246) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include "openvino/genai/text_streamer.hpp" #include "mediapipe/framework/calculator_graph.h" #pragma GCC diagnostic pop #pragma warning(pop) diff --git a/third_party/genai/genai_windows.BUILD b/third_party/genai/genai_windows.BUILD index ba3de7df7a..0f4cb24e09 100644 --- a/third_party/genai/genai_windows.BUILD +++ b/third_party/genai/genai_windows.BUILD @@ -21,7 +21,7 @@ package( cc_library( name = "genai_headers", hdrs = glob([ - "include/openvino/genai/*.*" + "include/openvino/**/*.*" ]), strip_include_prefix = "include", visibility = ["//visibility:public"], @@ -38,7 +38,7 @@ cc_import( cc_library( name = "genai", srcs = glob([ -# "bin\\intel64\\Release\\openvino_genai.dll", + "bin\\intel64\\Release\\openvino_genai.dll", "bin\\intel64\\Release\\icudt70.dll", "bin\\intel64\\Release\\icuuc70.dll", "bin\\intel64\\Release\\openvino_tokenizers.dll", @@ -46,7 +46,7 @@ cc_library( ]), visibility = ["//visibility:public"], deps = [ -# ":genai_lib", + ":genai_lib", ":genai_headers", ], ) diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 8ac3120c47..b01b418eed 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -189,8 +189,7 @@ if "!output_user_root!" neq "opt" ( powershell -Command "(gc -Path WORKSPACE) -replace '%genai_workspace%', '%genai_new_workspace%' | Set-Content -Path WORKSPACE" if !errorlevel! neq 0 exit /b !errorlevel! ) -:: Remove genai headers to be replaced by the ones from openvino_genai repository -rmdir /S /Q %BAZEL_SHORT_PATH%\%genai_dir%\runtime\include\openvino\genai + echo [INFO] GenAi installed: %BAZEL_SHORT_PATH%\%genai_dir% ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: From 0b2d025ebcaa3c8b4bcdae5b94cfcc89861b0a02 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 18 Sep 2025 14:02:39 +0200 Subject: [PATCH 07/29] Genai from source --- Dockerfile.redhat | 17 +++++++++-------- Dockerfile.ubuntu | 16 ++++++++-------- Makefile | 4 ++-- create_package.sh | 2 -- tools/update_ov/update_ov.py | 8 ++++---- 5 files changed, 23 insertions(+), 24 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 9ee92803a7..116827ed48 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -214,20 +214,21 @@ RUN if [ -f /opt/intel/openvino/samples/cpp/build_samples.sh ]; then /opt/intel ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/:/opt/opencv/lib/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ -WORKDIR /openvino_tokenizers/ +WORKDIR /openvino_genai/ -ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 -ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" +ARG ov_genai_branch=master # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ - git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + && cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + cp -r /openvino_genai/src/cpp/include/openvino/* /opt/intel/openvino/runtime/include ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ - cp -r python/* /opt/intel/openvino/python/ ; \ - cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ + cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ echo $'Metadata-Version: 1.0\nName: openvino-tokenizers\nVersion: 2025.4\nRequires-Python: >=3.9\nRequires-Dist: openvino~=2025.4.0' > /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info/METADATA ; \ - ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; + ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so # Build OpenVINO Model Server WORKDIR /ovms diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index b0d2aa1c85..c7745676bf 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -193,20 +193,20 @@ RUN if [ -f /opt/intel/openvino/samples/cpp/build_samples.sh ]; then /opt/intel # OpenVINO Tokenizers extension ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so -WORKDIR /openvino_tokenizers/ -ARG ov_tokenizers_branch=master -ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" +ARG ov_genai_branch=master # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ - git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ - cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + && cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + cp -r /openvino_genai/src/cpp/include/openvino/* /opt/intel/openvino/runtime/include ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ - cp -r python/* /opt/intel/openvino/python/ ; \ - cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ + cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ echo $'Metadata-Version: 1.0\nName: openvino-tokenizers\nVersion: 2025.4\nRequires-Python: >=3.9\nRequires-Dist: openvino~=2025.4.0' > /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info/METADATA ; \ - ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; + ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so # Add Nvidia dev tool if needed RUN apt-get update ; \ diff --git a/Makefile b/Makefile index c965e9b11c..c2b04f7103 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ FUZZER_BUILD ?= 0 OV_SOURCE_BRANCH ?= 05bf2133ac23f33957b87ce2eff184c2f8922f8d # master 2025/09/11 OV_CONTRIB_BRANCH ?= c39462ca8d7c550266dc70cdbfbe4fc8c5be0677 # master / 2024-10-31 -OV_TOKENIZERS_BRANCH ?= 196fb0dc4b2f5cdf5a993a43293011761534aaa4 # master 2025/09/11 +OV_GENAI_BRANCH ?= a4f4272e49ed363fb7fa79faf379f4287317832a # master 2025/09/11 OV_SOURCE_ORG ?= openvinotoolkit OV_CONTRIB_ORG ?= openvinotoolkit @@ -227,7 +227,7 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg BASE_IMAGE=$(BASE_IMAGE)\ --build-arg BASE_OS=$(BASE_OS)\ --build-arg ov_contrib_branch=$(OV_CONTRIB_BRANCH)\ - --build-arg ov_tokenizers_branch=$(OV_TOKENIZERS_BRANCH)\ + --build-arg ov_genai_branch=$(OV_GENAI_BRANCH)\ --build-arg INSTALL_RPMS_FROM_URL=$(INSTALL_RPMS_FROM_URL)\ --build-arg INSTALL_DRIVER_VERSION=$(INSTALL_DRIVER_VERSION)\ --build-arg RELEASE_BASE_IMAGE=$(BASE_IMAGE_RELEASE)\ diff --git a/create_package.sh b/create_package.sh index bc140b64ff..f3bccedc5d 100755 --- a/create_package.sh +++ b/create_package.sh @@ -21,8 +21,6 @@ mkdir -vp /ovms_release/bin mkdir -vp /ovms_release/lib mkdir -vp /ovms_release/lib/custom_nodes -if [ -f /openvino_tokenizers/build/src/libopenvino_tokenizers.so ]; then cp -v /openvino_tokenizers/build/src/libopenvino_tokenizers.so /ovms_release/lib/ ; fi - find /ovms/bazel-out/k8-*/bin -iname '*.so*' ! -type d ! -name "libgtest.so" ! -name "*params" ! -name "*.hana.*" ! -name "py_generate_pipeline.cpython*" ! -name "lib_node_*" ! -path "*test_python_binding*" ! -name "*libpython*" -exec cp -v {} /ovms_release/lib/ \; mv /ovms_release/lib/libcustom_node* /ovms_release/lib/custom_nodes/ cd /ovms_release/lib/ ; rm -f libcurl.so* diff --git a/tools/update_ov/update_ov.py b/tools/update_ov/update_ov.py index d23dbf8bf3..56c5538619 100644 --- a/tools/update_ov/update_ov.py +++ b/tools/update_ov/update_ov.py @@ -265,10 +265,10 @@ def update_openvino_genai(): # Update Makefile content with OpenVINO Tokenizers commit new_makefile_content = [] for line in makefile_content: - if "OV_TOKENIZERS_BRANCH ?=" in line: - print("Updating OV_TOKENIZERS_BRANCH") - indentation = line[:line.index("OV_TOKENIZERS_BRANCH ?=")] - new_line = f'{indentation}OV_TOKENIZERS_BRANCH ?= {tokenizers_commit} # master {tokenizers_commit_date}\n' + if "OV_GENAI_BRANCH ?=" in line: + print("Updating OV_GENAI_BRANCH") + indentation = line[:line.index("OV_GENAI_BRANCH ?=")] + new_line = f'{indentation}OV_GENAI_BRANCH ?= {tokenizers_commit} # master {tokenizers_commit_date}\n' print(new_line.lstrip()) new_makefile_content.append(new_line) else: From 8362d22b49dfec5d99ac18b04dacd41465beece6 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 18 Sep 2025 14:21:33 +0200 Subject: [PATCH 08/29] Add deps --- Dockerfile.redhat | 2 +- Dockerfile.ubuntu | 2 +- third_party/genai/BUILD | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 116827ed48..19c2704237 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -222,7 +222,7 @@ ARG ov_genai_branch=master RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ - && cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + && cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ cp -r /openvino_genai/src/cpp/include/openvino/* /opt/intel/openvino/runtime/include ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index c7745676bf..573614843f 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -200,7 +200,7 @@ ARG ov_genai_branch=master RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ - && cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ cp -r /openvino_genai/src/cpp/include/openvino/* /opt/intel/openvino/runtime/include ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ diff --git a/third_party/genai/BUILD b/third_party/genai/BUILD index 4dbfe81c7a..0f10bbc924 100644 --- a/third_party/genai/BUILD +++ b/third_party/genai/BUILD @@ -31,6 +31,7 @@ cc_library( name = "genai", srcs = glob([ "lib/intel64/libopenvino_genai.so", + "lib/intel64/libopenvino_genai.so.2540", "lib/intel64/libopenvino_tokenizers.so", ]), visibility = ["//visibility:public"], From 0b1aada95b8f6a3643479603954adcce5045a8a6 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 18 Sep 2025 14:21:50 +0200 Subject: [PATCH 09/29] Fix dockerfile --- Dockerfile.redhat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 19c2704237..6f9d6c3aa2 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -222,7 +222,7 @@ ARG ov_genai_branch=master RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ - && cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ + cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ cp -r /openvino_genai/src/cpp/include/openvino/* /opt/intel/openvino/runtime/include ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ From b7f82f44da253d2329a68a6788f5c2489879642d Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 18 Sep 2025 14:48:01 +0200 Subject: [PATCH 10/29] Fix include --- Dockerfile.redhat | 2 +- Dockerfile.ubuntu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 6f9d6c3aa2..1cffced79e 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -223,7 +223,7 @@ RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; t git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ - cp -r /openvino_genai/src/cpp/include/openvino/* /opt/intel/openvino/runtime/include ; \ + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 573614843f..e1dea30383 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -201,7 +201,7 @@ RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; t git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ - cp -r /openvino_genai/src/cpp/include/openvino/* /opt/intel/openvino/runtime/include ; \ + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ From 7c974fd999fb9ef690f2411358f74dd66971019e Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 18 Sep 2025 15:28:58 +0200 Subject: [PATCH 11/29] Fix spell and if --- Dockerfile.redhat | 4 ++-- Dockerfile.ubuntu | 4 ++-- spelling-whitelist.txt | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 1cffced79e..2deb0f1b9d 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -219,11 +219,11 @@ WORKDIR /openvino_genai/ ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" ARG ov_genai_branch=master # hadolint ignore=DL3003 -RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ - cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include ; \ + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index e1dea30383..81de260bc4 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -197,11 +197,11 @@ ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libop ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" ARG ov_genai_branch=master # hadolint ignore=DL3003 -RUN if [ "$ov_use_binary" == "0" ] && [[ $debug_bazel_flags == *"_py_off"* ]]; then true ; else exit 0 ; fi ; \ +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ - cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include ; \ + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index d6b81c0c06..0624485cc5 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -25,3 +25,5 @@ release_files/thirdparty-licenses/icu.LICENSE.txt:160: TaBE ==> table, tab release_files/thirdparty-licenses/libgt2.LICENSE.txt:1040: aheared ==> adhered release_files/thirdparty-licenses/libgt2.LICENSE.txt:1065: rouines ==> routines release_files/thirdparty-licenses/libgt2.LICENSE.txt:1083: publically ==> publicly +Dockerfile.redhat:228: thirdparty ==> third party, third-party +Dockerfile.ubuntu:206: thirdparty ==> third party, third-party From 1ba6ec88cdb270412ff765283f4b7404fca9db87 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 18 Sep 2025 15:43:03 +0200 Subject: [PATCH 12/29] Spell2 --- spelling-whitelist.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index 0624485cc5..50f0051f0b 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -1,3 +1,5 @@ +Dockerfile.redhat:228: thirdparty ==> third party, third-party +Dockerfile.ubuntu:206: thirdparty ==> third party, third-party client/common/resnet_labels.txt demos/common/python/classes.py demos/image_classification/go/labels.go @@ -25,5 +27,3 @@ release_files/thirdparty-licenses/icu.LICENSE.txt:160: TaBE ==> table, tab release_files/thirdparty-licenses/libgt2.LICENSE.txt:1040: aheared ==> adhered release_files/thirdparty-licenses/libgt2.LICENSE.txt:1065: rouines ==> routines release_files/thirdparty-licenses/libgt2.LICENSE.txt:1083: publically ==> publicly -Dockerfile.redhat:228: thirdparty ==> third party, third-party -Dockerfile.ubuntu:206: thirdparty ==> third party, third-party From 28f77cdcc89bfa70c66dcbe72ff8397dea50e376 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 18 Sep 2025 16:32:21 +0200 Subject: [PATCH 13/29] Add tokenizers flag --- Dockerfile.redhat | 2 +- Dockerfile.ubuntu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 2deb0f1b9d..1f2e656ebc 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -221,7 +221,7 @@ ARG ov_genai_branch=master # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ - cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=ON -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 81de260bc4..720391b3f4 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -199,7 +199,7 @@ ARG ov_genai_branch=master # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ - cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=ON -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ From 8ce18c78597a6a9ba70e487a8b73b3c4c44c1f09 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 19 Sep 2025 16:56:56 +0200 Subject: [PATCH 14/29] Add tokenizers back --- Dockerfile.redhat | 28 +++-- Dockerfile.ubuntu | 23 ++-- Makefile | 2 + create_package.sh | 2 + third_party/llm_engine/BUILD | 15 --- third_party/llm_engine/llm_engine.bzl | 174 -------------------------- 6 files changed, 40 insertions(+), 204 deletions(-) delete mode 100644 third_party/llm_engine/BUILD delete mode 100644 third_party/llm_engine/llm_engine.bzl diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 1f2e656ebc..992207afa1 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -214,22 +214,34 @@ RUN if [ -f /opt/intel/openvino/samples/cpp/build_samples.sh ]; then /opt/intel ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/:/opt/opencv/lib/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ -WORKDIR /openvino_genai/ + +WORKDIR /openvino_tokenizers/ +ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 +# hadolint ignore=DL3003 ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + # Install the openvino_tokenizers python bindings and copy to OpenVINO location + if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ + mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ + cp -r python/* /opt/intel/openvino/python/ ; \ + cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ + mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ + echo $'Metadata-Version: 1.0\nName: openvino-tokenizers\nVersion: 2025.4\nRequires-Python: >=3.9\nRequires-Dist: openvino~=2025.4.0' > /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info/METADATA ; \ + ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; \ + fi + +WORKDIR /openvino_genai/ ARG ov_genai_branch=master # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=ON -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ - cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ ; \ - mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ - cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ - mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ - echo $'Metadata-Version: 1.0\nName: openvino-tokenizers\nVersion: 2025.4\nRequires-Python: >=3.9\nRequires-Dist: openvino~=2025.4.0' > /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info/METADATA ; \ - ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so - + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ + # Build OpenVINO Model Server WORKDIR /ovms COPY .bazelrc .user.bazelr[c] .bazelversion WORKSPACE /ovms/ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 720391b3f4..82b42a71b9 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -195,18 +195,27 @@ ENV OpenVINO_DIR=/opt/intel/openvino/runtime/cmake ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" -ARG ov_genai_branch=master +ARG ov_tokenizers_branch=master # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ - git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ - cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=ON -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ - cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ - cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ ; \ + git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ + if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ - cp /openvino_genai/thirdparty/openvino_tokenizers/python/openvino_tokenizers/* /opt/intel/openvino/python/ ; \ + cp -r python/* /opt/intel/openvino/python/ ; \ + cp build/python/* /opt/intel/openvino/python/openvino_tokenizers/ ; \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info ; \ echo $'Metadata-Version: 1.0\nName: openvino-tokenizers\nVersion: 2025.4\nRequires-Python: >=3.9\nRequires-Dist: openvino~=2025.4.0' > /opt/intel/openvino/python/openvino_tokenizers-2025.4.dist-info/METADATA ; \ - ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so + ln -s /ovms/lib/libopenvino_tokenizers.so /opt/intel/openvino/python/openvino_tokenizers/lib/libopenvino_tokenizers.so ; \ + fi + +ARG ov_genai_branch=master +# hadolint ignore=DL3003 +RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ + git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ + cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ # Add Nvidia dev tool if needed RUN apt-get update ; \ diff --git a/Makefile b/Makefile index c2b04f7103..328a1d860b 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,7 @@ FUZZER_BUILD ?= 0 OV_SOURCE_BRANCH ?= 05bf2133ac23f33957b87ce2eff184c2f8922f8d # master 2025/09/11 OV_CONTRIB_BRANCH ?= c39462ca8d7c550266dc70cdbfbe4fc8c5be0677 # master / 2024-10-31 +OV_TOKENIZERS_BRANCH ?= 196fb0dc4b2f5cdf5a993a43293011761534aaa4 # master 2025/09/11 OV_GENAI_BRANCH ?= a4f4272e49ed363fb7fa79faf379f4287317832a # master 2025/09/11 OV_SOURCE_ORG ?= openvinotoolkit @@ -227,6 +228,7 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg BASE_IMAGE=$(BASE_IMAGE)\ --build-arg BASE_OS=$(BASE_OS)\ --build-arg ov_contrib_branch=$(OV_CONTRIB_BRANCH)\ + --build-arg ov_tokenizers_branch=$(OV_TOKENIZERS_BRANCH)\ --build-arg ov_genai_branch=$(OV_GENAI_BRANCH)\ --build-arg INSTALL_RPMS_FROM_URL=$(INSTALL_RPMS_FROM_URL)\ --build-arg INSTALL_DRIVER_VERSION=$(INSTALL_DRIVER_VERSION)\ diff --git a/create_package.sh b/create_package.sh index f3bccedc5d..324b7d0049 100755 --- a/create_package.sh +++ b/create_package.sh @@ -21,6 +21,8 @@ mkdir -vp /ovms_release/bin mkdir -vp /ovms_release/lib mkdir -vp /ovms_release/lib/custom_nodes +# TODO: decide if needed : if [ -f /openvino_tokenizers/build/src/libopenvino_tokenizers.so ]; then cp -v /openvino_tokenizers/build/src/libopenvino_tokenizers.so /ovms_release/lib/ ; fi + find /ovms/bazel-out/k8-*/bin -iname '*.so*' ! -type d ! -name "libgtest.so" ! -name "*params" ! -name "*.hana.*" ! -name "py_generate_pipeline.cpython*" ! -name "lib_node_*" ! -path "*test_python_binding*" ! -name "*libpython*" -exec cp -v {} /ovms_release/lib/ \; mv /ovms_release/lib/libcustom_node* /ovms_release/lib/custom_nodes/ cd /ovms_release/lib/ ; rm -f libcurl.so* diff --git a/third_party/llm_engine/BUILD b/third_party/llm_engine/BUILD deleted file mode 100644 index 9c4ccf9af2..0000000000 --- a/third_party/llm_engine/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# diff --git a/third_party/llm_engine/llm_engine.bzl b/third_party/llm_engine/llm_engine.bzl deleted file mode 100644 index f9e4d9b65d..0000000000 --- a/third_party/llm_engine/llm_engine.bzl +++ /dev/null @@ -1,174 +0,0 @@ -# -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") - -def _is_windows(ctx): - return ctx.os.name.lower().find("windows") != -1 - -def llm_engine(): - llm_engine_repository(name="_llm_engine") - new_git_repository( - name = "llm_engine", - remote = "https://github.com/openvinotoolkit/openvino.genai", - commit = "a4f4272e49ed363fb7fa79faf379f4287317832a", # master 2025-09-11 Check available memory before allocating KV-cache. - build_file = "@_llm_engine//:BUILD", - init_submodules = True, - recursive_init_submodules = True, - patch_args = ["-p1"], - ) - # when using local repository manually run: git submodule update --recursive - #native.new_local_repository( - # name = "llm_engine", - # path = "/openvino.genai", - # build_file = "@_llm_engine//:BUILD", - #) - -def _impl(repository_ctx): - http_proxy = repository_ctx.os.environ.get("http_proxy", "") - if http_proxy == "": - http_proxy = repository_ctx.os.environ.get("HTTP_PROXY", "") - - https_proxy = repository_ctx.os.environ.get("https_proxy", "") - if https_proxy == "": - https_proxy = repository_ctx.os.environ.get("HTTPS_PROXY", "") - - OpenVINO_DIR = repository_ctx.os.environ.get("OpenVINO_DIR", "") - - if _is_windows(repository_ctx): - icudt = "icudt70" - icuuc = "icuuc70" - tokenizers = "openvino_tokenizers" - lib_name = "openvino_genai" - OpenVINO_DIR = OpenVINO_DIR.replace("\\", "\\\\").replace("/", "\\\\") - out_dll_dir_win = "out_dll_dir = \"runtime/bin/Release\"," - out_lib_dir = "out_lib_dir = \"runtime/lib/Release\"" - out_static = "out_interface_libs = [\"{lib_name}.lib\"],".format(lib_name=lib_name) - out_libs = "out_shared_libs = [\"{lib_name}.dll\"],".format(lib_name=lib_name) - cache_entries = """ - "CMAKE_POSITION_INDEPENDENT_CODE": "ON", - "CMAKE_CXX_FLAGS": " /guard:cf /GS -s -D_GLIBCXX_USE_CXX11_ABI=1", - "CMAKE_LIBRARY_OUTPUT_DIRECTORY": "runtime/bin/Release", - "WIN32": "True", - "X86_64": "True", - "BUILD_TOKENIZERS": "OFF", - "ENABLE_SAMPLES": "OFF", - "ENABLE_TOOLS": "OFF", - "ENABLE_TESTS": "OFF", - "ENABLE_XGRAMMAR": "ON", - """ - jobs_param = "\"-j 8\"" # on Windows we do not need to specify number of jobs, it's set to all available cores number - else: - lib_name = "libopenvino_genai" - out_dll_dir_win = "" - out_lib_dir = "out_lib_dir = \"runtime/lib/intel64\"" - out_static = "" - out_libs = "out_shared_libs = [\"{lib_name}.so.2540\"],".format(lib_name=lib_name) - cache_entries = """ - "BUILD_SHARED_LIBS": "OFF", - "CMAKE_POSITION_INDEPENDENT_CODE": "ON", - "CMAKE_CXX_FLAGS": " -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized", - "CMAKE_ARCHIVE_OUTPUT_DIRECTORY": "lib", - "ENABLE_SYSTEM_ICU": "True", - "BUILD_TOKENIZERS": "OFF", - "ENABLE_SAMPLES": "OFF", - "ENABLE_TOOLS": "OFF", - "ENABLE_TESTS": "OFF", - "ENABLE_XGRAMMAR": "ON", - """ - jobs_param = "\"-j 8\"" # on Linux we need to specify jobs number, by default it's set to 1 - - # Note we need to escape '{/}' by doubling them due to call to format - build_file_content = """ -load("@rules_foreign_cc//foreign_cc:cmake.bzl", "cmake") -load("@bazel_skylib//rules:common_settings.bzl", "string_flag") -visibility = ["//visibility:public"] - -config_setting( - name = "dbg", - values = {{"compilation_mode": "dbg"}}, -) - -config_setting( - name = "opt", - values = {{"compilation_mode": "opt"}}, -) - -filegroup( - name = "all_srcs", - srcs = glob(["**"]), - visibility = ["//visibility:public"], -) - -build_release = {{"CMAKE_BUILD_TYPE": "Release"}} -build_debug = {{"CMAKE_BUILD_TYPE": "Debug"}} - -cmake( - name = "llm_engine_cmake", - build_args = [ - "--verbose", - "--", # <- Pass remaining options to the native tool. - # https://github.com/bazelbuild/rules_foreign_cc/issues/329 - # there is no elegant parallel compilation support - lets go with default - CORES + 2 for ninja - {jobs_param} - ], - cache_entries = {{ - {cache_entries} - }} | select({{ - "//conditions:default": dict( - build_release - ), - # Debug does not build - #":dbg": dict( - # build_debug - #), - }}), - env = {{ - "OpenVINO_DIR": "{OpenVINO_DIR}", - "http_proxy": "{http_proxy}", - "https_proxy": "{https_proxy}", - }}, - lib_source = ":all_srcs", - out_include_dir = "runtime/include", - {out_lib_dir}, - {out_libs} - {out_static} - {out_dll_dir_win} - tags = ["requires-network"], - visibility = ["//visibility:public"], - lib_name = "{lib_name}", - deps = [ - "@ovms//third_party:openvino", - ] -) - -cc_library( - name = "llm_engine", - deps = [ - "@ovms//third_party:openvino", - ":llm_engine_cmake", - ], - visibility = ["//visibility:public"], -) -""" - repository_ctx.file("BUILD", build_file_content.format(OpenVINO_DIR=OpenVINO_DIR, http_proxy=http_proxy, https_proxy=https_proxy, - out_dll_dir_win=out_dll_dir_win, out_lib_dir=out_lib_dir, lib_name=lib_name, out_libs=out_libs, cache_entries=cache_entries, out_static=out_static, - jobs_param=jobs_param)) - -llm_engine_repository = repository_rule( - implementation = _impl, - local=True, -) From 00baf0c5537633b4aa7d8c6cb34de97a57b56bdf Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 19 Sep 2025 17:30:43 +0200 Subject: [PATCH 15/29] Fix sdl --- Dockerfile.redhat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 992207afa1..79a8e51ca8 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -217,9 +217,8 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/: WORKDIR /openvino_tokenizers/ ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 -# hadolint ignore=DL3003 - ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" +# hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; \ git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ From 510edcf267919649ae39a16167d2c8614046248a Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 22 Sep 2025 12:15:32 +0200 Subject: [PATCH 16/29] Fix tokenizers --- Dockerfile.redhat | 6 ++++-- Dockerfile.ubuntu | 6 ++++-- Makefile | 10 +++++++--- create_package.sh | 2 -- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 79a8e51ca8..8adccbc0d1 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -217,10 +217,11 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/: WORKDIR /openvino_tokenizers/ ARG ov_tokenizers_branch=85be884a69f10270703f81f970a5ee596a4c8df7 +ARG ov_tokenizers_org=openvinotoolkit ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; \ - git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ + git clone https://github.com/$ov_tokenizers_org/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE="${VERBOSE_LOGS}" -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ # Install the openvino_tokenizers python bindings and copy to OpenVINO location if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ @@ -234,9 +235,10 @@ RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; \ WORKDIR /openvino_genai/ ARG ov_genai_branch=master +ARG ov_genai_org=openvinotoolkit # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ - git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=ON -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 82b42a71b9..f70624492e 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -196,9 +196,10 @@ ENV OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libop ARG SDL_OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=deprecated-declarations -Wuninitialized" ARG ov_tokenizers_branch=master +ARG ov_tokenizers_org=openvinotoolkit # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ - git clone https://github.com/openvinotoolkit/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ + git clone https://github.com/$ov_tokenizers_org/openvino_tokenizers.git /openvino_tokenizers && cd /openvino_tokenizers && git checkout $ov_tokenizers_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -S ./ -B ./build/ && cmake --build ./build/ --parallel $JOBS && cp /openvino_tokenizers/build/src/lib*.so /opt/intel/openvino/runtime/lib/intel64/ ; \ if ! [[ $debug_bazel_flags == *"_py_off"* ]]; then \ mkdir -p /opt/intel/openvino/python/openvino_tokenizers/lib ; \ @@ -210,9 +211,10 @@ RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debu fi ARG ov_genai_branch=master +ARG ov_genai_org=openvinotoolkit # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ - git clone https://github.com/openvinotoolkit/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ + git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS="${SDL_OPS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ diff --git a/Makefile b/Makefile index 328a1d860b..c4ef3bfae3 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,8 @@ OV_GENAI_BRANCH ?= a4f4272e49ed363fb7fa79faf379f4287317832a # master 2025/09/11 OV_SOURCE_ORG ?= openvinotoolkit OV_CONTRIB_ORG ?= openvinotoolkit +OV_GENAI_ORG ?= openvinotoolkit +OV_TOKENIZERS_ORG ?= openvinotoolkit TEST_LLM_PATH ?= "src/test/llm_testing" GPU_MODEL_PATH ?= "/tmp/face_detection_adas" @@ -213,7 +215,12 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg no_proxy=$(NO_PROXY)\ --build-arg ov_source_branch=$(OV_SOURCE_BRANCH)\ --build-arg ov_source_org=$(OV_SOURCE_ORG)\ + --build-arg ov_genai_org=$(OV_GENAI_ORG)\ + --build-arg ov_tokenizers_org=$(OV_TOKENIZERS_ORG)\ --build-arg ov_contrib_org=$(OV_CONTRIB_ORG)\ + --build-arg ov_contrib_branch=$(OV_CONTRIB_BRANCH)\ + --build-arg ov_tokenizers_branch=$(OV_TOKENIZERS_BRANCH)\ + --build-arg ov_genai_branch=$(OV_GENAI_BRANCH)\ --build-arg ov_use_binary=$(OV_USE_BINARY)\ --build-arg DLDT_PACKAGE_URL=$(DLDT_PACKAGE_URL)\ --build-arg CHECK_COVERAGE=$(CHECK_COVERAGE)\ @@ -227,9 +234,6 @@ BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ --build-arg PROJECT_VERSION=$(PROJECT_VERSION)\ --build-arg BASE_IMAGE=$(BASE_IMAGE)\ --build-arg BASE_OS=$(BASE_OS)\ - --build-arg ov_contrib_branch=$(OV_CONTRIB_BRANCH)\ - --build-arg ov_tokenizers_branch=$(OV_TOKENIZERS_BRANCH)\ - --build-arg ov_genai_branch=$(OV_GENAI_BRANCH)\ --build-arg INSTALL_RPMS_FROM_URL=$(INSTALL_RPMS_FROM_URL)\ --build-arg INSTALL_DRIVER_VERSION=$(INSTALL_DRIVER_VERSION)\ --build-arg RELEASE_BASE_IMAGE=$(BASE_IMAGE_RELEASE)\ diff --git a/create_package.sh b/create_package.sh index 324b7d0049..f3bccedc5d 100755 --- a/create_package.sh +++ b/create_package.sh @@ -21,8 +21,6 @@ mkdir -vp /ovms_release/bin mkdir -vp /ovms_release/lib mkdir -vp /ovms_release/lib/custom_nodes -# TODO: decide if needed : if [ -f /openvino_tokenizers/build/src/libopenvino_tokenizers.so ]; then cp -v /openvino_tokenizers/build/src/libopenvino_tokenizers.so /ovms_release/lib/ ; fi - find /ovms/bazel-out/k8-*/bin -iname '*.so*' ! -type d ! -name "libgtest.so" ! -name "*params" ! -name "*.hana.*" ! -name "py_generate_pipeline.cpython*" ! -name "lib_node_*" ! -path "*test_python_binding*" ! -name "*libpython*" -exec cp -v {} /ovms_release/lib/ \; mv /ovms_release/lib/libcustom_node* /ovms_release/lib/custom_nodes/ cd /ovms_release/lib/ ; rm -f libcurl.so* From c9d60d03c1136531f7f122d708851e9727f5a225 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 23 Sep 2025 13:12:00 +0200 Subject: [PATCH 17/29] Add test env --- .bazelrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.bazelrc b/.bazelrc index cbe0a96d08..a5dede7272 100644 --- a/.bazelrc +++ b/.bazelrc @@ -260,6 +260,8 @@ test --experimental_ui_max_stdouterr_bytes=304857600 # envs below are required to succesfully run binaries in bazel as bazel is very strict in sandboxing test:linux --test_env LD_LIBRARY_PATH=/opt/opencv/lib/:/opt/intel/openvino/runtime/lib/intel64/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ +# genai hardcodes path during build time so without this so is not visible +test:linux --test_env OPENVINO_TOKENIZERS_PATH_GENAI=/opt/intel/openvino/runtime/lib/intel64/libopenvino_tokenizers.so test:linux --test_env PYTHONPATH=/opt/intel/openvino/python:/ovms/bazel-bin/src/python/binding test:linux --test_env no_proxy=localhost From 983ab94ae3a0d771ebd757d44f7454a20fff3c02 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 26 Sep 2025 16:26:47 +0200 Subject: [PATCH 18/29] Fix redhat --- Dockerfile.redhat | 2 +- Makefile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 8adccbc0d1..9a1e86704f 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -239,7 +239,7 @@ ARG ov_genai_org=openvinotoolkit # hadolint ignore=DL3003 RUN if [ "$ov_use_binary" == "0" ]; then true ; else exit 0 ; fi ; if ! [[ $debug_bazel_flags == *"py_off"* ]]; then true ; else exit 0 ; fi ; \ git clone https://github.com/$ov_genai_org/openvino.genai /openvino_genai && cd /openvino_genai && git checkout $ov_genai_branch && git submodule update --init --recursive ; \ - cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=ON -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ + cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DCMAKE_CXX_FLAGS=" ${SDL_OPS} ${LTO_CXX_FLAGS} " -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DENABLE_SYSTEM_ICU="True" -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON -S ./ -B ./build/ ; \ cmake --build ./build/ --parallel $JOBS && cp /openvino_genai/build/openvino_genai/lib*.so* /opt/intel/openvino/runtime/lib/intel64/ ; \ cp -r /openvino_genai/src/cpp/include/* /opt/intel/openvino/runtime/include/ diff --git a/Makefile b/Makefile index f6b7717d65..5d0b7bc5bd 100644 --- a/Makefile +++ b/Makefile @@ -140,6 +140,7 @@ ifeq ($(findstring ubuntu,$(BASE_OS)),ubuntu) TARGET_DISTRO_PARAMS = " --//:distro=ubuntu" else ifeq ($(findstring redhat,$(BASE_OS)),redhat) TARGET_DISTRO_PARAMS = " --//:distro=redhat" + OV_USE_BINARY = 0 else $(error BASE_OS must be either ubuntu or redhat) endif From 8fb557107d4c1509151a795427a657956c4b0612 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 26 Sep 2025 17:28:15 +0200 Subject: [PATCH 19/29] Fix dockerfile --- Dockerfile.redhat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 9a1e86704f..8cc508d788 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -164,7 +164,7 @@ ARG ov_source_branch=c01cd93e24d1cd78bfbb401eed51c08fb93e0816 ARG ov_contrib_branch=master ARG ov_source_org=openvinotoolkit ARG ov_contrib_org=openvinotoolkit -ARG ov_use_binary=0 +ENV ov_use_binary=0 ARG DLDT_PACKAGE_URL ARG TEMP_DIR=/tmp/openvino_installer ARG CMAKE_BUILD_TYPE=Release From 39cc8efdcfbd7671a8bf8707e4e59c0ed99cea4e Mon Sep 17 00:00:00 2001 From: rasapala Date: Mon, 29 Sep 2025 15:54:09 +0200 Subject: [PATCH 20/29] Add windows build from source --- windows_create_package.bat | 14 +++- windows_install_build_dependencies.bat | 109 ++++++++++++++++++++++++- 2 files changed, 118 insertions(+), 5 deletions(-) diff --git a/windows_create_package.bat b/windows_create_package.bat index 8d9474ee62..22c08ee2bd 100644 --- a/windows_create_package.bat +++ b/windows_create_package.bat @@ -106,10 +106,16 @@ md %license_dest% if !errorlevel! neq 0 exit /b !errorlevel! copy C:\opt\opencv_4.12.0\etc\licenses\* %license_dest% if !errorlevel! neq 0 exit /b !errorlevel! -copy C:\%output_user_root%\openvino\docs\licensing\LICENSE %license_dest%openvino.LICENSE.txt -if !errorlevel! neq 0 exit /b !errorlevel! -copy C:\%output_user_root%\openvino\docs\licensing\LICENSE-GENAI %license_dest%LICENSE-GENAI.txt -if !errorlevel! neq 0 exit /b !errorlevel! +if "%USE_OV_BINARY%"=="1" ( + copy C:\%output_user_root%\openvino\docs\licensing\LICENSE %license_dest%openvino.LICENSE.txt + if !errorlevel! neq 0 exit /b !errorlevel! + copy C:\%output_user_root%\openvino\docs\licensing\LICENSE-GENAI %license_dest%LICENSE-GENAI.txt + if !errorlevel! neq 0 exit /b !errorlevel! +) +else ( + copy openvino\LICENSE %license_dest%openvino.LICENSE.txt + if !errorlevel! neq 0 exit /b !errorlevel! +) copy %cd%\release_files\LICENSE %cd%\dist\windows\ovms\ if !errorlevel! neq 0 exit /b !errorlevel! diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 36ade3e0c5..801615e02f 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -129,6 +129,20 @@ IF /I EXIST %bash_path% ( echo [INFO] Msys installed in: %msys_path% ) +:: Set default USE_OV_BINARY if not set +if "%USE_OV_BINARY%"=="" ( + set "USE_OV_BINARY=1" +) + +set "genai_zip=%BAZEL_SHORT_PATH%\%genai_ver%" +set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" +set "genai_new_workspace=C:\\%output_user_root%\\openvino\\runtime" + +echo [INFO] USE_OV_BINARY=%USE_OV_BINARY% +IF "%USE_OV_BINARY%"=="0" ( + goto :install_openvino_from_src +) + ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::: Install in c:\PR-XXXX\ section started - once per build, reinstalled only with expunge clean :::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: @@ -190,8 +204,101 @@ if "!output_user_root!" neq "opt" ( if !errorlevel! neq 0 exit /b !errorlevel! ) -echo [INFO] GenAi installed: %BAZEL_SHORT_PATH%\%genai_dir% +echo [INFO] GenAI installed: %BAZEL_SHORT_PATH%\%genai_dir% +goto :finished_openvino + +:install_openvino_from_src +IF /I EXIST %BAZEL_SHORT_PATH%\openvino ( + rmdir /S /Q %BAZEL_SHORT_PATH%\openvino +) +if "%OV_SOURCE_BRANCH%"=="" ( + set "OV_SOURCE_BRANCH=4a90bedcdd64dc4d6fee54cf03de6c5617b35a48" +) +if "%OV_SOURCE_ORG%"=="" ( + set "OV_SOURCE_ORG=openvinotoolkit" +) +if "%TOKENIZER_SOURCE_ORG%"=="" ( + set "TOKENIZER_SOURCE_ORG=openvinotoolkit" +) +if "%TOKENIZER_SOURCE_BRANCH%"=="" ( + set "TOKENIZER_SOURCE_BRANCH=bd47b33bcae913c59dcbe7e67ff52dbdf826ac32" +) +if "%GENAI_SOURCE_ORG%"=="" ( + set "GENAI_SOURCE_ORG=openvinotoolkit" +) +if "%GENAI_SOURCE_BRANCH%"=="" ( + set "GENAI_SOURCE_BRANCH=845ba50e83e3f2f436d3794044a87f767f76bfd7" +) + +IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_src ( + git clone https://github.com/%OV_SOURCE_ORG%/openvino %BAZEL_SHORT_PATH%\openvino_src +) + +set "BACK_CWD=%cd%" +cd %BAZEL_SHORT_PATH%\openvino_src +git fetch origin +git checkout %OV_SOURCE_BRANCH% +if !errorlevel! neq 0 exit /b !errorlevel! +git submodule update --init --recursive +if !errorlevel! neq 0 exit /b !errorlevel! +IF /I NOT EXIST build ( + mkdir build +) +cd build +set "TBB_DIR=" +cmake -G "Visual Studio 17 2022" -DENABLE_SAMPLES=OFF -DENABLE_INTEL_NPU_PROTOPIPE=OFF .. +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --build . --config Release --verbose -j +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --install . --config Release --prefix %BAZEL_SHORT_PATH%\openvino +if !errorlevel! neq 0 exit /b !errorlevel! +call %BAZEL_SHORT_PATH%\openvino\setupvars.bat + +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +::::::::::::::::::::::: OpenVINO Tokenizers + +IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_tokenizers_src ( + git clone https://github.com/%TOKENIZER_SOURCE_ORG%/openvino_tokenizers.git %BAZEL_SHORT_PATH%\openvino_tokenizers_src +) +cd %BAZEL_SHORT_PATH%\openvino_tokenizers_src +git fetch origin +git checkout %TOKENIZER_SOURCE_BRANCH% +if !errorlevel! neq 0 exit /b !errorlevel! +IF /I NOT EXIST build ( + mkdir build +) +cd build +cmake -DCMAKE_BUILD_TYPE=Release .. +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --build . --config Release --verbose -j +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --install . --config Release --prefix %BAZEL_SHORT_PATH%\openvino +if !errorlevel! neq 0 exit /b !errorlevel! + +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +::::::::::::::::::::::: OpenVINO GenAI + +IF /I NOT EXIST %BAZEL_SHORT_PATH%\openvino_genai_src ( + git clone https://github.com/%GENAI_SOURCE_ORG%/openvino.genai.git %BAZEL_SHORT_PATH%\openvino_genai_src +) +cd %BAZEL_SHORT_PATH%\openvino_genai_src +git fetch origin +git checkout %GENAI_SOURCE_BRANCH% +if !errorlevel! neq 0 exit /b !errorlevel! +IF /I NOT EXIST build ( + mkdir build +) +cd build +cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TOKENIZERS=OFF -DENABLE_SAMPLES=OFF -DENABLE_TOOLS=OFF -DENABLE_TESTS=OFF -DENABLE_XGRAMMAR=ON .. +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --build . --config Release --verbose -j +if !errorlevel! neq 0 exit /b !errorlevel! +cmake --install . --config Release --prefix %BAZEL_SHORT_PATH%\openvino +if !errorlevel! neq 0 exit /b !errorlevel! +echo [INFO] OpenVINO from source installed: %BAZEL_SHORT_PATH%\openvino +cd !BACK_CWD! +:finished_openvino ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::: OpenCL headers echo [INFO] Installing OpenCL headers ... From d4c1573c3934d0bb7d4a0e8f249e1760f2c552c3 Mon Sep 17 00:00:00 2001 From: rasapala Date: Mon, 29 Sep 2025 16:23:22 +0200 Subject: [PATCH 21/29] Test source --- windows_create_package.bat | 7 +++---- windows_install_build_dependencies.bat | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/windows_create_package.bat b/windows_create_package.bat index 22c08ee2bd..171a329172 100644 --- a/windows_create_package.bat +++ b/windows_create_package.bat @@ -106,14 +106,13 @@ md %license_dest% if !errorlevel! neq 0 exit /b !errorlevel! copy C:\opt\opencv_4.12.0\etc\licenses\* %license_dest% if !errorlevel! neq 0 exit /b !errorlevel! -if "%USE_OV_BINARY%"=="1" ( +IF "%USE_OV_BINARY%"=="1" ( copy C:\%output_user_root%\openvino\docs\licensing\LICENSE %license_dest%openvino.LICENSE.txt if !errorlevel! neq 0 exit /b !errorlevel! copy C:\%output_user_root%\openvino\docs\licensing\LICENSE-GENAI %license_dest%LICENSE-GENAI.txt if !errorlevel! neq 0 exit /b !errorlevel! -) -else ( - copy openvino\LICENSE %license_dest%openvino.LICENSE.txt +) ELSE ( + copy C:\%output_user_root%\openvino\licenses %license_dest% if !errorlevel! neq 0 exit /b !errorlevel! ) diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 801615e02f..644f008ed9 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -131,7 +131,7 @@ IF /I EXIST %bash_path% ( :: Set default USE_OV_BINARY if not set if "%USE_OV_BINARY%"=="" ( - set "USE_OV_BINARY=1" + set "USE_OV_BINARY=0" ) set "genai_zip=%BAZEL_SHORT_PATH%\%genai_ver%" From e9c5b6c85b0c2ac76e982ccb3eedd6917d871e83 Mon Sep 17 00:00:00 2001 From: rasapala Date: Tue, 30 Sep 2025 10:31:22 +0200 Subject: [PATCH 22/29] Fix workspace changes --- windows_install_build_dependencies.bat | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 644f008ed9..6c957334a4 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -134,9 +134,13 @@ if "%USE_OV_BINARY%"=="" ( set "USE_OV_BINARY=0" ) -set "genai_zip=%BAZEL_SHORT_PATH%\%genai_ver%" set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" set "genai_new_workspace=C:\\%output_user_root%\\openvino\\runtime" +:: Replace path to GenAi in ovms WORKSPACE file +if "!output_user_root!" neq "opt" ( + powershell -Command "(gc -Path WORKSPACE) -replace '%genai_workspace%', '%genai_new_workspace%' | Set-Content -Path WORKSPACE" + if !errorlevel! neq 0 exit /b !errorlevel! +) echo [INFO] USE_OV_BINARY=%USE_OV_BINARY% IF "%USE_OV_BINARY%"=="0" ( @@ -148,7 +152,7 @@ IF "%USE_OV_BINARY%"=="0" ( ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -::::::::::::::::::::::: GENAI/OPENVINO - reinstalled per build trigger +::::::::::::::::::::::: GENAI/OPENVINO install from ZIP - reinstalled per build trigger :: Set default GENAI_PACKAGE_URL if not set if "%GENAI_PACKAGE_URL%"=="" ( set "GENAI_PACKAGE_URL=https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2025.4.0.0.dev20250923/openvino_genai_windows_2025.4.0.0.dev20250923_x86_64.zip" @@ -161,8 +165,6 @@ for %%F in ("%GENAI_PACKAGE_URL%") do set "genai_ver=%%~nxF" for %%F in ("%genai_ver%") do set "genai_dir=%%~nF" set "genai_zip=%BAZEL_SHORT_PATH%\%genai_ver%" -set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" -set "genai_new_workspace=C:\\%output_user_root%\\openvino\\runtime" echo [INFO] Installing GenAI: %genai_dir% ... :: Download GenAi @@ -198,12 +200,6 @@ IF /I EXIST %BAZEL_SHORT_PATH%\openvino ( mklink /d %BAZEL_SHORT_PATH%\openvino %BAZEL_SHORT_PATH%\%genai_dir% if !errorlevel! neq 0 exit /b !errorlevel! -:: Replace path to GenAi in ovms WORKSPACE file -if "!output_user_root!" neq "opt" ( - powershell -Command "(gc -Path WORKSPACE) -replace '%genai_workspace%', '%genai_new_workspace%' | Set-Content -Path WORKSPACE" - if !errorlevel! neq 0 exit /b !errorlevel! -) - echo [INFO] GenAI installed: %BAZEL_SHORT_PATH%\%genai_dir% goto :finished_openvino From d122b9aea47eb1ce7742df13036e757bc3d865c6 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 30 Sep 2025 16:31:25 +0200 Subject: [PATCH 23/29] Fix test and ov_bin=0 --- Makefile | 2 +- src/test/embeddingsnode_test.cpp | 78 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5d0b7bc5bd..4e82e0f7cd 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ OV_TOKENIZERS_ORG ?= openvinotoolkit TEST_LLM_PATH ?= "src/test/llm_testing" GPU_MODEL_PATH ?= "/tmp/face_detection_adas" -OV_USE_BINARY ?= 1 +OV_USE_BINARY = 0 APT_OV_PACKAGE ?= openvino-2022.1.0 # opt, dbg: BAZEL_BUILD_TYPE ?= opt diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index 978742f5b2..b0530a09c5 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -19,6 +19,7 @@ #include #include "../http_rest_api_handler.hpp" +#include "../servablemanagermodule.hpp" #include "../server.hpp" #include "rapidjson/document.h" #include "test_http_utils.hpp" @@ -399,9 +400,39 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values( "embeddings", "embeddings_ov")); +static bool isMpReady(const std::string name) { + ovms::Server& server = ovms::Server::instance(); + const ovms::Module* servableModule = server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); + if (!servableModule) { + return false; + } + ModelManager* manager = &dynamic_cast(servableModule)->getServableManager(); + auto mediapipeGraphDefinition = manager->getMediapipeFactory().findDefinitionByName(name); + if (!mediapipeGraphDefinition) { + return false; + } + return mediapipeGraphDefinition->getStatus().isAvailable(); +} + +static bool waitMpReady(const std::string name) { + SPDLOG_TRACE("waitMpReady:{}", name); + auto start = std::chrono::high_resolution_clock::now(); + while (!isMpReady(name) && + (std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - start).count() < SERVER_START_FROM_CONFIG_TIMEOUT_SECONDS/4)) { + std::this_thread::sleep_for(std::chrono::microseconds(1000)); + } + + return isMpReady(name); +} + class EmbeddingsExtensionTest : public ::testing::Test { protected: static std::unique_ptr t; + static std::unique_ptr t1; + static std::unique_ptr t2; + static std::unique_ptr t3; + static std::unique_ptr t4; + static std::unique_ptr t5; public: std::unique_ptr handler; @@ -436,6 +467,38 @@ class EmbeddingsExtensionTest : public ::testing::Test { EXPECT_EQ(EXIT_SUCCESS, server.start(argc, argv)); })); EnsureServerStartedWithTimeout(server, 15); + + t1.reset(new std::thread([]() { + const std::string embeddings{"embeddings"}; + ASSERT_EQ(waitMpReady(embeddings), true); + })); + t2.reset(new std::thread([]() { + const std::string embeddings_no_norm{"embeddings_no_norm"}; + ASSERT_EQ(waitMpReady(embeddings_no_norm), true); + })); + t3.reset(new std::thread([]() { + const std::string embeddings_ov{"embeddings_ov"}; + ASSERT_EQ(waitMpReady(embeddings_ov), true); + })); + t4.reset(new std::thread([]() { + const std::string embeddings_ov_no_norm{"embeddings_ov_no_norm"}; + ASSERT_EQ(waitMpReady(embeddings_ov_no_norm), true); + })); + t5.reset(new std::thread([]() { + const std::string embeddings_ov_relative{"embeddings_ov_relative"}; + ASSERT_EQ(waitMpReady(embeddings_ov_relative), true); + })); + + if(t1) + t1->join(); + if(t2) + t2->join(); + if(t3) + t3->join(); + if(t4) + t4->join(); + if(t5) + t5->join(); } void SetUp() { @@ -457,6 +520,16 @@ class EmbeddingsExtensionTest : public ::testing::Test { server.setShutdownRequest(1); t->join(); server.setShutdownRequest(0); + if(t1) + t1->join(); + if(t2) + t2->join(); + if(t3) + t3->join(); + if(t4) + t4->join(); + if(t5) + t5->join(); } void TearDown() { @@ -467,6 +540,11 @@ class EmbeddingsExtensionTest : public ::testing::Test { } }; std::unique_ptr EmbeddingsExtensionTest::t; +std::unique_ptr EmbeddingsExtensionTest::t1; +std::unique_ptr EmbeddingsExtensionTest::t2; +std::unique_ptr EmbeddingsExtensionTest::t3; +std::unique_ptr EmbeddingsExtensionTest::t4; +std::unique_ptr EmbeddingsExtensionTest::t5; TEST_F(EmbeddingsExtensionTest, simplePositive) { std::string requestBody = R"( { From 4a423d39959b49218c5415023fedaf0a7fcd8aa7 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 30 Sep 2025 16:35:09 +0200 Subject: [PATCH 24/29] Style --- src/test/embeddingsnode_test.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index b0530a09c5..6dd15548e6 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -418,7 +418,7 @@ static bool waitMpReady(const std::string name) { SPDLOG_TRACE("waitMpReady:{}", name); auto start = std::chrono::high_resolution_clock::now(); while (!isMpReady(name) && - (std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - start).count() < SERVER_START_FROM_CONFIG_TIMEOUT_SECONDS/4)) { + (std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - start).count() < SERVER_START_FROM_CONFIG_TIMEOUT_SECONDS / 4)) { std::this_thread::sleep_for(std::chrono::microseconds(1000)); } @@ -489,15 +489,15 @@ class EmbeddingsExtensionTest : public ::testing::Test { ASSERT_EQ(waitMpReady(embeddings_ov_relative), true); })); - if(t1) + if (t1) t1->join(); - if(t2) + if (t2) t2->join(); - if(t3) + if (t3) t3->join(); - if(t4) + if (t4) t4->join(); - if(t5) + if (t5) t5->join(); } @@ -520,15 +520,15 @@ class EmbeddingsExtensionTest : public ::testing::Test { server.setShutdownRequest(1); t->join(); server.setShutdownRequest(0); - if(t1) + if (t1) t1->join(); - if(t2) + if (t2) t2->join(); - if(t3) + if (t3) t3->join(); - if(t4) + if (t4) t4->join(); - if(t5) + if (t5) t5->join(); } From 86ed5f28249adef51dc200cae344efa6f959a5f5 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 1 Oct 2025 08:33:41 +0200 Subject: [PATCH 25/29] test ubuntu with bin on --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4e82e0f7cd..5d0b7bc5bd 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ OV_TOKENIZERS_ORG ?= openvinotoolkit TEST_LLM_PATH ?= "src/test/llm_testing" GPU_MODEL_PATH ?= "/tmp/face_detection_adas" -OV_USE_BINARY = 0 +OV_USE_BINARY ?= 1 APT_OV_PACKAGE ?= openvino-2022.1.0 # opt, dbg: BAZEL_BUILD_TYPE ?= opt From 83b15cd92fdbd718085f6b926ac8b71abf0649bc Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 1 Oct 2025 10:50:20 +0200 Subject: [PATCH 26/29] Fix test --- Makefile | 2 +- src/test/embeddingsnode_test.cpp | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 5d0b7bc5bd..39370b8840 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ FUZZER_BUILD ?= 0 OV_SOURCE_BRANCH ?= 4a90bedcdd64dc4d6fee54cf03de6c5617b35a48 # master 2025/09/23 OV_CONTRIB_BRANCH ?= c39462ca8d7c550266dc70cdbfbe4fc8c5be0677 # master / 2024-10-31 -OV_TOKENIZERS_BRANCH ?= bd47b33bcae913c59dcbe7e67ff52dbdf826ac32 # master 2025/09/11 +OV_TOKENIZERS_BRANCH ?= bd47b33bcae913c59dcbe7e67ff52dbdf826ac32 # master 2025/09/18 OV_GENAI_BRANCH ?= 845ba50e83e3f2f436d3794044a87f767f76bfd7 # master 2025/09/23 OV_SOURCE_ORG ?= openvinotoolkit diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index 6dd15548e6..736107de92 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -489,15 +489,15 @@ class EmbeddingsExtensionTest : public ::testing::Test { ASSERT_EQ(waitMpReady(embeddings_ov_relative), true); })); - if (t1) + if (t1->joinable()) t1->join(); - if (t2) + if (t2->joinable()) t2->join(); - if (t3) + if (t3->joinable()) t3->join(); - if (t4) + if (t4->joinable()) t4->join(); - if (t5) + if (t5->joinable()) t5->join(); } @@ -520,15 +520,15 @@ class EmbeddingsExtensionTest : public ::testing::Test { server.setShutdownRequest(1); t->join(); server.setShutdownRequest(0); - if (t1) + if (t1->joinable()) t1->join(); - if (t2) + if (t2->joinable()) t2->join(); - if (t3) + if (t3->joinable()) t3->join(); - if (t4) + if (t4->joinable()) t4->join(); - if (t5) + if (t5->joinable()) t5->join(); } From 8febedbbe16635a40570f11e20c5fe609eef05db Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 2 Oct 2025 07:15:10 +0200 Subject: [PATCH 27/29] Remove tokenizer from linking --- create_package.sh | 3 +++ third_party/genai/BUILD | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/create_package.sh b/create_package.sh index f3bccedc5d..d18159af91 100755 --- a/create_package.sh +++ b/create_package.sh @@ -21,6 +21,9 @@ mkdir -vp /ovms_release/bin mkdir -vp /ovms_release/lib mkdir -vp /ovms_release/lib/custom_nodes +# Do not link this tokenizer lib as it as old protobuf sentencepiece symbols the conflict with new protobuf from ovsm +if [ "$ov_use_binary" == "0" ] ; then cp -v /openvino_tokenizers/build/src/libopenvino_tokenizers.so /ovms_release/lib/ ; fi + find /ovms/bazel-out/k8-*/bin -iname '*.so*' ! -type d ! -name "libgtest.so" ! -name "*params" ! -name "*.hana.*" ! -name "py_generate_pipeline.cpython*" ! -name "lib_node_*" ! -path "*test_python_binding*" ! -name "*libpython*" -exec cp -v {} /ovms_release/lib/ \; mv /ovms_release/lib/libcustom_node* /ovms_release/lib/custom_nodes/ cd /ovms_release/lib/ ; rm -f libcurl.so* diff --git a/third_party/genai/BUILD b/third_party/genai/BUILD index 0f10bbc924..0fb74fc1ac 100644 --- a/third_party/genai/BUILD +++ b/third_party/genai/BUILD @@ -32,7 +32,6 @@ cc_library( srcs = glob([ "lib/intel64/libopenvino_genai.so", "lib/intel64/libopenvino_genai.so.2540", - "lib/intel64/libopenvino_tokenizers.so", ]), visibility = ["//visibility:public"], deps = [":genai_headers"], From a0d714aa7bac5a0402fc6508e9cf7bd3c18b8f43 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 2 Oct 2025 07:47:31 +0200 Subject: [PATCH 28/29] SEt default build from bin windows --- windows_install_build_dependencies.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows_install_build_dependencies.bat b/windows_install_build_dependencies.bat index 6c957334a4..3f902faa73 100644 --- a/windows_install_build_dependencies.bat +++ b/windows_install_build_dependencies.bat @@ -131,7 +131,7 @@ IF /I EXIST %bash_path% ( :: Set default USE_OV_BINARY if not set if "%USE_OV_BINARY%"=="" ( - set "USE_OV_BINARY=0" + set "USE_OV_BINARY=1" ) set "genai_workspace=C:\\\\opt\\\\openvino\\\\runtime" From 8cdf220fdeb2639130505ab59f5a389dd6260fce Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 2 Oct 2025 10:23:27 +0200 Subject: [PATCH 29/29] Remove header --- src/llm/BUILD | 20 ++++--------- src/llm/apis/openai_completions.hpp | 6 ++-- src/llm/generation_config_header.hpp | 28 ------------------- .../base_generation_config_builder.cpp | 2 +- .../base_generation_config_builder.hpp | 2 +- src/llm/io_processing/base_output_parser.hpp | 2 +- .../generation_config_builder.hpp | 2 +- .../hermes3/generation_config_builder.cpp | 2 +- .../llama3/generation_config_builder.cpp | 2 +- .../phi4/generation_config_builder.cpp | 2 +- .../continuous_batching/llm_executor.hpp | 3 +- .../continuous_batching/servable.hpp | 3 +- .../servable_initializer.cpp | 3 +- .../language_model/legacy/legacy_executor.hpp | 3 +- src/llm/language_model/legacy/servable.hpp | 3 +- .../legacy/servable_initializer.cpp | 2 +- .../continuous_batching/servable.hpp | 2 ++ .../legacy/legacy_executor.hpp | 3 +- .../visual_language_model/legacy/servable.hpp | 3 +- .../legacy/servable_initializer.cpp | 2 +- src/test/llm/assisted_decoding_test.cpp | 2 +- src/test/llm/llmnode_test.cpp | 2 +- src/test/llm/llmtemplate_test.cpp | 2 +- 23 files changed, 38 insertions(+), 63 deletions(-) delete mode 100644 src/llm/generation_config_header.hpp diff --git a/src/llm/BUILD b/src/llm/BUILD index 7ee20609b6..9b69ae9268 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -17,14 +17,6 @@ load("@mediapipe//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") load("//:common_settings.bzl", "ovms_cc_library", "PYBIND_DEPS", "COPTS_PYTHON") -ovms_cc_library( - name = "genai_proxy", - hdrs = ["generation_config_header.hpp"], - srcs = [], - deps = ["//third_party:genai"], - visibility = ["//visibility:public"], -) - ovms_cc_library( name = "llmcalculator", srcs = ["http_llm_calculator.cc"], @@ -36,7 +28,7 @@ ovms_cc_library( "//src:libovmsprofiler", ":genai_servables", "//src:httppayload", - ":genai_proxy",], + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON, alwayslink = 1, # needed, so the calculator can be registered by MediaPipe @@ -66,7 +58,7 @@ ovms_cc_library( "@stb//:image", ":openai_request", ":output_parsers", - ":genai_proxy",], + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -75,7 +67,7 @@ ovms_cc_library( name = "openai_request", hdrs = ["apis/openai_request.hpp"], srcs = [], - deps = [":genai_proxy",], + deps = ["//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -110,7 +102,7 @@ ovms_cc_library( "@com_github_tencent_rapidjson//:rapidjson", "//src:libovmslogging", "//src:libovmsstring_utils", - ":genai_proxy",], + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -129,7 +121,7 @@ ovms_cc_library( deps = [ ":openai_request", "//src:libovmslogging", - ":genai_proxy",], + "//third_party:genai",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON ) @@ -176,7 +168,7 @@ ovms_cc_library( ":generation_config_builders", "//src:httppayload", "//src:libhttpclientconnection", - ":genai_proxy",] + select({ + "//third_party:genai",] + select({ "//:disable_python": [], "//:not_disable_python" : [":py_jinja_template_processor"], }), diff --git a/src/llm/apis/openai_completions.hpp b/src/llm/apis/openai_completions.hpp index 5dd9200f51..55461d90c9 100644 --- a/src/llm/apis/openai_completions.hpp +++ b/src/llm/apis/openai_completions.hpp @@ -24,8 +24,11 @@ #include #include +#include +#include +#include #include - +#include #pragma warning(push) #pragma warning(disable : 6313) #include @@ -36,7 +39,6 @@ #include "absl/status/status.h" #pragma warning(pop) #include "../io_processing/output_parser.hpp" -#include "../generation_config_header.hpp" #include "openai_request.hpp" using namespace rapidjson; diff --git a/src/llm/generation_config_header.hpp b/src/llm/generation_config_header.hpp deleted file mode 100644 index c91cfc1fff..0000000000 --- a/src/llm/generation_config_header.hpp +++ /dev/null @@ -1,28 +0,0 @@ -//***************************************************************************** -// Copyright 2025 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#pragma warning(push) -#pragma warning(disable : 4505) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wreorder" -#include -#include -#include -#include -#include -#include -#pragma GCC diagnostic pop -#pragma warning(pop) diff --git a/src/llm/io_processing/base_generation_config_builder.cpp b/src/llm/io_processing/base_generation_config_builder.cpp index f65572e94d..5ae2c55a85 100644 --- a/src/llm/io_processing/base_generation_config_builder.cpp +++ b/src/llm/io_processing/base_generation_config_builder.cpp @@ -16,8 +16,8 @@ #include #include +#include #include "base_generation_config_builder.hpp" -#include "../generation_config_header.hpp" namespace ovms { void BaseGenerationConfigBuilder::setStructuralTagsConfig(const ov::genai::StructuralTagsConfig& structuralTagsConfig) { diff --git a/src/llm/io_processing/base_generation_config_builder.hpp b/src/llm/io_processing/base_generation_config_builder.hpp index 733d769f72..3af0f7c54d 100644 --- a/src/llm/io_processing/base_generation_config_builder.hpp +++ b/src/llm/io_processing/base_generation_config_builder.hpp @@ -16,9 +16,9 @@ #pragma once #include +#include #include #include "../apis/openai_request.hpp" -#include "../generation_config_header.hpp" namespace ovms { diff --git a/src/llm/io_processing/base_output_parser.hpp b/src/llm/io_processing/base_output_parser.hpp index c5ce319ff9..d6500265f0 100644 --- a/src/llm/io_processing/base_output_parser.hpp +++ b/src/llm/io_processing/base_output_parser.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include #include @@ -28,7 +29,6 @@ #include #pragma warning(pop) -#include "../generation_config_header.hpp" #include "partial_json_builder.hpp" namespace ovms { diff --git a/src/llm/io_processing/generation_config_builder.hpp b/src/llm/io_processing/generation_config_builder.hpp index 5d45c5c279..9dd3289596 100644 --- a/src/llm/io_processing/generation_config_builder.hpp +++ b/src/llm/io_processing/generation_config_builder.hpp @@ -18,9 +18,9 @@ #include #include #include +#include #include #include "base_generation_config_builder.hpp" -#include "../generation_config_header.hpp" #include "phi4/generation_config_builder.hpp" #include "llama3/generation_config_builder.hpp" #include "hermes3/generation_config_builder.hpp" diff --git a/src/llm/io_processing/hermes3/generation_config_builder.cpp b/src/llm/io_processing/hermes3/generation_config_builder.cpp index 2ea50d6b51..c42b876d83 100644 --- a/src/llm/io_processing/hermes3/generation_config_builder.cpp +++ b/src/llm/io_processing/hermes3/generation_config_builder.cpp @@ -16,8 +16,8 @@ #include #include +#include -#include "../../generation_config_header.hpp" #include "generation_config_builder.hpp" namespace ovms { diff --git a/src/llm/io_processing/llama3/generation_config_builder.cpp b/src/llm/io_processing/llama3/generation_config_builder.cpp index d9f17c6301..fa8b7f4058 100644 --- a/src/llm/io_processing/llama3/generation_config_builder.cpp +++ b/src/llm/io_processing/llama3/generation_config_builder.cpp @@ -16,9 +16,9 @@ #include #include +#include #include "generation_config_builder.hpp" -#include "../../generation_config_header.hpp" namespace ovms { diff --git a/src/llm/io_processing/phi4/generation_config_builder.cpp b/src/llm/io_processing/phi4/generation_config_builder.cpp index 0df7e9f4ac..1142cfc4c9 100644 --- a/src/llm/io_processing/phi4/generation_config_builder.cpp +++ b/src/llm/io_processing/phi4/generation_config_builder.cpp @@ -16,9 +16,9 @@ #include #include +#include #include "generation_config_builder.hpp" -#include "../../generation_config_header.hpp" namespace ovms { diff --git a/src/llm/language_model/continuous_batching/llm_executor.hpp b/src/llm/language_model/continuous_batching/llm_executor.hpp index 11dc00174f..f396d78dc3 100644 --- a/src/llm/language_model/continuous_batching/llm_executor.hpp +++ b/src/llm/language_model/continuous_batching/llm_executor.hpp @@ -24,7 +24,8 @@ #include #include -#include "../../generation_config_header.hpp" +#include + #include "../../../logging.hpp" #include "../../../profiler.hpp" diff --git a/src/llm/language_model/continuous_batching/servable.hpp b/src/llm/language_model/continuous_batching/servable.hpp index 23711daa1b..def574e009 100644 --- a/src/llm/language_model/continuous_batching/servable.hpp +++ b/src/llm/language_model/continuous_batching/servable.hpp @@ -17,7 +17,8 @@ #include #include -#include "../../generation_config_header.hpp" +#include + #include "../../servable.hpp" #include "src/llm/llm_calculator.pb.h" diff --git a/src/llm/language_model/continuous_batching/servable_initializer.cpp b/src/llm/language_model/continuous_batching/servable_initializer.cpp index ac01198803..ec0d011dd4 100644 --- a/src/llm/language_model/continuous_batching/servable_initializer.cpp +++ b/src/llm/language_model/continuous_batching/servable_initializer.cpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include #include @@ -29,7 +31,6 @@ #pragma GCC diagnostic pop #pragma warning(pop) -#include "../../generation_config_header.hpp" #include "../../../json_parser.hpp" #include "../../../logging.hpp" #include "../../../mediapipe_internal/mediapipe_utils.hpp" diff --git a/src/llm/language_model/legacy/legacy_executor.hpp b/src/llm/language_model/legacy/legacy_executor.hpp index 83ac51d41a..5bb3365bd2 100644 --- a/src/llm/language_model/legacy/legacy_executor.hpp +++ b/src/llm/language_model/legacy/legacy_executor.hpp @@ -25,7 +25,8 @@ #include #include -#include "../../generation_config_header.hpp" +#include "openvino/genai/llm_pipeline.hpp" + #include "../../../logging.hpp" #include "../../../profiler.hpp" diff --git a/src/llm/language_model/legacy/servable.hpp b/src/llm/language_model/legacy/servable.hpp index 820e21baa2..b2f0aa01d1 100644 --- a/src/llm/language_model/legacy/servable.hpp +++ b/src/llm/language_model/legacy/servable.hpp @@ -17,7 +17,8 @@ #include #include -#include "../../generation_config_header.hpp" +#include "openvino/genai/llm_pipeline.hpp" + #include "../../servable.hpp" #include "legacy_executor.hpp" #include "src/llm/llm_calculator.pb.h" diff --git a/src/llm/language_model/legacy/servable_initializer.cpp b/src/llm/language_model/legacy/servable_initializer.cpp index f1012b9d45..4ee7d4820a 100644 --- a/src/llm/language_model/legacy/servable_initializer.cpp +++ b/src/llm/language_model/legacy/servable_initializer.cpp @@ -18,6 +18,7 @@ #include #include +#include "openvino/genai/llm_pipeline.hpp" #include #include @@ -29,7 +30,6 @@ #pragma GCC diagnostic pop #pragma warning(pop) -#include "../../generation_config_header.hpp" #include "../../../json_parser.hpp" #include "../../../logging.hpp" #include "../../../mediapipe_internal/mediapipe_utils.hpp" diff --git a/src/llm/visual_language_model/continuous_batching/servable.hpp b/src/llm/visual_language_model/continuous_batching/servable.hpp index 7f468a9906..88c78e3dcb 100644 --- a/src/llm/visual_language_model/continuous_batching/servable.hpp +++ b/src/llm/visual_language_model/continuous_batching/servable.hpp @@ -18,6 +18,8 @@ #include #include +#include + #include "../../language_model/continuous_batching/servable.hpp" namespace ovms { diff --git a/src/llm/visual_language_model/legacy/legacy_executor.hpp b/src/llm/visual_language_model/legacy/legacy_executor.hpp index 013ef10cdd..b2e33ab7cf 100644 --- a/src/llm/visual_language_model/legacy/legacy_executor.hpp +++ b/src/llm/visual_language_model/legacy/legacy_executor.hpp @@ -25,7 +25,8 @@ #include #include -#include "../../generation_config_header.hpp" +#include + #include "../../../logging.hpp" #include "../../../profiler.hpp" diff --git a/src/llm/visual_language_model/legacy/servable.hpp b/src/llm/visual_language_model/legacy/servable.hpp index cee3d803f9..0e77be8010 100644 --- a/src/llm/visual_language_model/legacy/servable.hpp +++ b/src/llm/visual_language_model/legacy/servable.hpp @@ -18,7 +18,8 @@ #include #include -#include "../../generation_config_header.hpp" +#include + #include "../../servable.hpp" #include "legacy_executor.hpp" #include "src/llm/llm_calculator.pb.h" diff --git a/src/llm/visual_language_model/legacy/servable_initializer.cpp b/src/llm/visual_language_model/legacy/servable_initializer.cpp index 7d809343ca..5fddcbc98e 100644 --- a/src/llm/visual_language_model/legacy/servable_initializer.cpp +++ b/src/llm/visual_language_model/legacy/servable_initializer.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -33,7 +34,6 @@ #include "../../../logging.hpp" #include "../../../mediapipe_internal/mediapipe_utils.hpp" #include "../../../status.hpp" -#include "../../generation_config_header.hpp" #include "servable.hpp" #include "servable_initializer.hpp" diff --git a/src/test/llm/assisted_decoding_test.cpp b/src/test/llm/assisted_decoding_test.cpp index 2db1ed6a80..86f4f26e57 100644 --- a/src/test/llm/assisted_decoding_test.cpp +++ b/src/test/llm/assisted_decoding_test.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #if (PYTHON_DISABLE == 0) #pragma warning(push) @@ -36,7 +37,6 @@ #include "../../http_status_code.hpp" #include "../../json_parser.hpp" #include "../../llm/apis/openai_completions.hpp" -#include "../../llm/generation_config_header.hpp" #include "../../llm/language_model/continuous_batching/servable.hpp" #include "../../llm/language_model/continuous_batching/llm_executor.hpp" #include "../../llm/text_utils.hpp" diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 4cf1b0a7d1..1a96fabb99 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #if (PYTHON_DISABLE == 0) #pragma warning(push) @@ -36,7 +37,6 @@ #include "../../http_status_code.hpp" #include "../../json_parser.hpp" #include "../../llm/apis/openai_completions.hpp" -#include "../../llm/generation_config_header.hpp" #include "../../llm/language_model/continuous_batching/llm_executor.hpp" #include "../../llm/language_model/continuous_batching/servable.hpp" #include "../../llm/servable.hpp" diff --git a/src/test/llm/llmtemplate_test.cpp b/src/test/llm/llmtemplate_test.cpp index 2f01b73050..848d795a05 100644 --- a/src/test/llm/llmtemplate_test.cpp +++ b/src/test/llm/llmtemplate_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #pragma warning(push) #pragma warning(disable : 6326 28182 6011 28020) @@ -31,7 +32,6 @@ #include "../../http_payload.hpp" #include "../../http_rest_api_handler.hpp" #include "../../httpservermodule.hpp" -#include "../../llm/generation_config_header.hpp" #include "../../llm/language_model/continuous_batching/servable.hpp" #include "../../llm/language_model/continuous_batching/servable_initializer.hpp" #include "../../llm/py_jinja_template_processor.hpp"