From d302a9ca9c0eb42772bc1e8d4a43f349fb2c19a0 Mon Sep 17 00:00:00 2001 From: Arpana Jindal Date: Thu, 18 Jun 2026 14:41:27 +0530 Subject: [PATCH 01/10] pipeline enhancement --- take-away/src/core/pipeline_runner.py | 30 +++++++++++++--------- take-away/src/parallel/station_worker.py | 32 +++++++++++++++--------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/take-away/src/core/pipeline_runner.py b/take-away/src/core/pipeline_runner.py index 20091415..44aa3817 100755 --- a/take-away/src/core/pipeline_runner.py +++ b/take-away/src/core/pipeline_runner.py @@ -4,6 +4,10 @@ import logging from minio import Minio +RTSP_DEFAULT_LATENCY = os.getenv("RTSP_LATENCY", "500") +CAPTURE_FPS = int(os.getenv("CAPTURE_FPS", "10")) +queue_params = "max-size-time=0 max-size-bytes=0 max-size-buffers=200 leaky=no" + # Configure logging logging.basicConfig( level=logging.INFO, @@ -19,7 +23,7 @@ def build_gstreamer_pipeline(source_type: str, source: str) -> str: elif source_type == "rtsp": source = normalize_rtsp_url(source) - src = f"rtspsrc location={source} protocols=tcp latency=200" + src = f"rtspsrc location={source} protocols=tcp latency={RTSP_DEFAULT_LATENCY} timeout=5000000 drop-on-latency=true retry=3" elif source_type == "webcam": src = f"v4l2src device={source}" @@ -31,17 +35,19 @@ def build_gstreamer_pipeline(source_type: str, source: str) -> str: logger.error(f"Unsupported source_type: {source_type}") raise ValueError(f"Unsupported source_type: {source_type}") - pipeline = ( - f"{src} " - "! decodebin " - "! videoconvert " - "! video/x-raw,format=BGR " - "! videorate " - "! video/x-raw,framerate=10/1 " - "! queue max-size-time=0 max-size-bytes=0 max-size-buffers=1000 leaky=no " - "! gvapython module=frame_pipeline function=process_frame " - "! fakesink sync=true" # sync=true ensures real-time playback so OCR can keep up - ) + pipeline = ( + f"{src} " + "! rtph264depay " + "! h264parse config-interval=-1 " + "! avdec_h264 " + "! videoconvert " + "! video/x-raw,format=BGR " + "! videorate " + f"! video/x-raw,framerate={CAPTURE_FPS}/1 " + f"! queue {queue_params} " + "! gvapython module=frame_pipeline function=process_frame " + "! fakesink sync=false" + ) logger.info(f"GStreamer pipeline built: {pipeline[:100]}...") return pipeline diff --git a/take-away/src/parallel/station_worker.py b/take-away/src/parallel/station_worker.py index 92a41cfd..e9cae5d2 100755 --- a/take-away/src/parallel/station_worker.py +++ b/take-away/src/parallel/station_worker.py @@ -49,10 +49,10 @@ class PipelineConfig: """Configuration for GStreamer pipeline behavior.""" - # RTSP source hardening - LOW LATENCY for fast connection - rtsp_latency_ms: int = 0 # Zero buffering (was 200ms) + # RTSP source hardening - balanced latency for stable connection + rtsp_latency_ms: int = 500 # 500ms buffering for stable RTSP rtsp_retry_count: int = 50 # Conservative retry count for RTSP reconnection - rtsp_timeout_us: int = 2000000 # 2 seconds + rtsp_timeout_us: int = 5000000 # 5 seconds rtsp_keepalive: bool = True rtsp_drop_on_latency: bool = True rtsp_protocols: str = "tcp" @@ -81,9 +81,9 @@ def from_dict(cls, config: Dict) -> 'PipelineConfig': """Create config from dictionary, using defaults for missing values.""" pipeline_cfg = config.get('pipeline', {}) return cls( - rtsp_latency_ms=pipeline_cfg.get('rtsp_latency_ms', 200), + rtsp_latency_ms=pipeline_cfg.get('rtsp_latency_ms', 500), rtsp_retry_count=pipeline_cfg.get('rtsp_retry_count', 50), - rtsp_timeout_us=pipeline_cfg.get('rtsp_timeout_us', 2000000), + rtsp_timeout_us=pipeline_cfg.get('rtsp_timeout_us', 5000000), rtsp_keepalive=pipeline_cfg.get('rtsp_keepalive', True), rtsp_drop_on_latency=pipeline_cfg.get('rtsp_drop_on_latency', True), rtsp_protocols=pipeline_cfg.get('rtsp_protocols', 'tcp'), @@ -932,7 +932,8 @@ def _verify_rtsp_stream_quick(self, timeout_sec: int = 3) -> bool: [ 'timeout', str(timeout_sec), 'gst-launch-1.0', '-e', - 'rtspsrc', f'location={self.rtsp_url}', 'protocols=tcp', 'latency=0', + 'rtspsrc', f'location={self.rtsp_url}', 'protocols=tcp', 'latency=500', + 'timeout=5000000', # 5 second RTSP timeout '!', 'fakesink', 'sync=false' ], capture_output=True, @@ -1056,27 +1057,34 @@ def _build_persistent_gstreamer_pipeline(self) -> str: # NOTE: 1fps is the recommended rate for CPU-based EasyOCR + YOLO processing. capture_fps = int(os.environ.get("CAPTURE_FPS", "10")) + queue_params = "max-size-time=0 max-size-bytes=0 max-size-buffers=200 leaky=no" + # Check if source is RTSP - use optimized low-latency rtspsrc if self.rtsp_url.startswith("rtsp://"): # LOW-LATENCY RTSP PIPELINE - # - latency=0: Zero buffering (default is 2000ms!) + # - latency: Configurable buffering (default 500ms) # - buffer-mode=0: Auto/slave mode for minimal delay # - ntp-sync=false: Skip NTP synchronization # - do-rtcp=false: Disable RTCP feedback loop # - protocols=tcp: Force TCP for reliability - # - retry=5: Quick retry on connection issues + # - retry=3: Quick retry on connection issues + # - timeout: Configurable RTSP timeout (default 5s) + # - drop-on-latency: Drop frames when buffer full to avoid stalls + # - h264parse config-interval=-1: Send SPS/PPS with every keyframe # - queue: Buffer frames to prevent drops during slow OCR processing # max-size-buffers=200 holds ~200 frames (enough for 200s at 1fps) pipeline = ( - f'rtspsrc location={self.rtsp_url} latency=0 buffer-mode=0 ' - 'protocols=tcp ntp-sync=false do-rtcp=false retry=5 ' + f'rtspsrc location={self.rtsp_url} latency={cfg.rtsp_latency_ms} buffer-mode=0 ' + f'protocols=tcp ntp-sync=false do-rtcp=false retry=3 ' + f'timeout={cfg.rtsp_timeout_us} drop-on-latency={str(cfg.rtsp_drop_on_latency).lower()} ' '! rtph264depay ' + '! h264parse config-interval=-1 ' '! avdec_h264 ' '! videoconvert ' '! video/x-raw,format=BGR ' '! videorate ' f'! video/x-raw,framerate={capture_fps}/1 ' - '! queue max-size-time=0 max-size-bytes=0 max-size-buffers=200 leaky=no ' + f'! queue {queue_params} ' f'! gvapython module={frame_pipeline_module} function=process_frame ' '! fakesink sync=false' ) @@ -1094,7 +1102,7 @@ def _build_persistent_gstreamer_pipeline(self) -> str: '! video/x-raw,format=BGR ' '! videorate ' f'! video/x-raw,framerate={capture_fps}/1 ' - '! queue max-size-time=0 max-size-bytes=0 max-size-buffers=200 leaky=no ' + f'! queue {queue_params} ' f'! gvapython module={frame_pipeline_module} function=process_frame ' '! fakesink sync=false' ) From 8c1e958c5ba281ae971e483a928e1b6d49fea173 Mon Sep 17 00:00:00 2001 From: intel Date: Fri, 19 Jun 2026 12:55:16 +0530 Subject: [PATCH 02/10] pipeline changes --- take-away/src/core/pipeline_runner.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/take-away/src/core/pipeline_runner.py b/take-away/src/core/pipeline_runner.py index 44aa3817..71217a7c 100755 --- a/take-away/src/core/pipeline_runner.py +++ b/take-away/src/core/pipeline_runner.py @@ -35,19 +35,17 @@ def build_gstreamer_pipeline(source_type: str, source: str) -> str: logger.error(f"Unsupported source_type: {source_type}") raise ValueError(f"Unsupported source_type: {source_type}") - pipeline = ( - f"{src} " - "! rtph264depay " - "! h264parse config-interval=-1 " - "! avdec_h264 " - "! videoconvert " - "! video/x-raw,format=BGR " - "! videorate " - f"! video/x-raw,framerate={CAPTURE_FPS}/1 " - f"! queue {queue_params} " - "! gvapython module=frame_pipeline function=process_frame " - "! fakesink sync=false" - ) + pipeline = ( + f"{src} " + "! decodebin " + "! videoconvert " + "! video/x-raw,format=BGR " + "! videorate " + f"! video/x-raw,framerate={CAPTURE_FPS}/1 " + f"! queue {queue_params} " + "! gvapython module=frame_pipeline function=process_frame " + "! fakesink sync=true" + ) logger.info(f"GStreamer pipeline built: {pipeline[:100]}...") return pipeline From 5a1fc9d59f849367815291d776d5c88fc7cfc8e5 Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Fri, 17 Jul 2026 10:06:50 +0530 Subject: [PATCH 03/10] Add MiniCPM model support and optional Hugging Face token configuration --- dine-in/.env.example | 8 ++- dine-in/README.md | 5 +- ovms-service/README.md | 19 +++++-- ovms-service/setup_models.sh | 102 +++++++++++++++++++++++++++-------- take-away/.env.example | 5 ++ 5 files changed, 113 insertions(+), 26 deletions(-) diff --git a/dine-in/.env.example b/dine-in/.env.example index 805ba052..dcb9a7ba 100755 --- a/dine-in/.env.example +++ b/dine-in/.env.example @@ -17,12 +17,18 @@ LOG_LEVEL=INFO # Service Endpoints # ----------------------------------------------------------------------------- OVMS_ENDPOINT=http://ovms-vlm:8000 -OVMS_MODEL_NAME=Qwen/Qwen2.5-VL-7B-Instruct +OVMS_MODEL_NAME=openbmb/MiniCPM-V-4_5 VLM_PRECISION=int8 TARGET_DEVICE=GPU SEMANTIC_SERVICE_ENDPOINT=http://semantic-service:8080 API_TIMEOUT=60 +# Optional Hugging Face authentication for gated models. +# Leave empty for public models (for example Qwen/Qwen2.5-VL-7B-Instruct). +# setup_models.sh also accepts HUGGINGFACE_HUB_TOKEN as an alternative name. +HF_TOKEN= + + # ----------------------------------------------------------------------------- # Benchmark Mode Configuration (In-App Scaling) # ----------------------------------------------------------------------------- diff --git a/dine-in/README.md b/dine-in/README.md index be0978e3..4867ba53 100644 --- a/dine-in/README.md +++ b/dine-in/README.md @@ -63,9 +63,12 @@ cd ../dine-in This step: -- Downloads Qwen2.5-VL-7B-Instruct from HuggingFace (~7 GB) +- Reads `OVMS_MODEL_NAME` from `dine-in/.env` (default: `openbmb/MiniCPM-V-4_5`) +- Downloads the selected model from HuggingFace - Converts to OpenVINO™ INT8 format +If the selected model is gated on Hugging Face, add `HF_TOKEN=` to `dine-in/.env` (or run `huggingface-cli login`). Public models such as `Qwen/Qwen2.5-VL-7B-Instruct` do not require authentication. + ### 3. Build and Start **Option A: Using Registry Images (default)** diff --git a/ovms-service/README.md b/ovms-service/README.md index 8e244bf6..f5a45432 100755 --- a/ovms-service/README.md +++ b/ovms-service/README.md @@ -21,7 +21,7 @@ ovms-service/ ### Prerequisites -1. **Disk space**: ~8 GB for Qwen2.5-VL-7B-Instruct-ov-int8 model (int8 quantization) +1. **Disk space**: ~8-12 GB depending on selected VLM and precision ### Export Model @@ -37,11 +37,24 @@ bash ovms-service/setup_models.sh --app dine-in This will: - Download `export_model.py` and install its dependencies automatically -- Download the model from HuggingFace +- Read `OVMS_MODEL_NAME` from the selected app `.env` (`take-away/.env` or `dine-in/.env`) +- Download that model from HuggingFace - Convert to OpenVINO™ IR format with int8 quantization -- Save to `ovms-service/models/Qwen/Qwen2.5-VL-7B-Instruct/` +- Save to `ovms-service/models//` - Generate `graph.pbtxt` for OVMS configuration +Supported values for `OVMS_MODEL_NAME`: + +- `Qwen/Qwen2.5-VL-7B-Instruct` +- `openbmb/MiniCPM-V-4_5` + +Optional authentication for gated Hugging Face models: + +- Add `HF_TOKEN=` (or `HUGGINGFACE_HUB_TOKEN=`) to the selected app `.env` +- Or run `huggingface-cli login` + +Public models (for example Qwen) can be downloaded without authentication. + > **ℹ Low-RAM systems:** Set `export CACHE_SIZE=2` before running `setup_models.sh` if you are on a 16 GB system. For first-time export, a 48–64 GB host is recommended to avoid OOM. See [Tuning the KV Cache Size](#tuning-the-kv-cache-size). ## Running OVMS diff --git a/ovms-service/setup_models.sh b/ovms-service/setup_models.sh index 0a5fbbcc..18313e51 100755 --- a/ovms-service/setup_models.sh +++ b/ovms-service/setup_models.sh @@ -32,12 +32,13 @@ if [ "${APP}" != "take-away" ] && [ "${APP}" != "dine-in" ]; then fi ############################################### -# HARD CODED MODEL REGISTRY +# SUPPORTED MODEL REGISTRY # key = model_name passed to --model_name (also used as path under MODELS_DIR) # value = HuggingFace source passed to --source_model ############################################### -declare -A MODEL_SOURCES -MODEL_SOURCES["Qwen/Qwen2.5-VL-7B-Instruct"]="Qwen/Qwen2.5-VL-7B-Instruct" +declare -A SUPPORTED_MODEL_SOURCES +SUPPORTED_MODEL_SOURCES["Qwen/Qwen2.5-VL-7B-Instruct"]="Qwen/Qwen2.5-VL-7B-Instruct" +SUPPORTED_MODEL_SOURCES["openbmb/MiniCPM-V-4_5"]="openbmb/MiniCPM-V-4_5" POTENTIAL_SOURCE_DIRS=( "${HOME}/ovms-vlm/models" @@ -48,20 +49,59 @@ POTENTIAL_SOURCE_DIRS=( ############################################### # LOAD CONFIG FROM APP-SPECIFIC .env ############################################### +read_env_value() { + local key="$1" + local file="$2" + grep -E "^${key}=" "${file}" 2>/dev/null | head -1 | cut -d'=' -f2- | tr -d '"\r' +} + ENV_FILE="${PROJECT_ROOT}/${APP}/.env" echo "App: ${APP} → reading config from ${ENV_FILE}" if [ -f "${ENV_FILE}" ]; then - OVMS_MODEL_NAME_ENV=$(grep -E '^OVMS_MODEL_NAME=' "${ENV_FILE}" | head -1 | cut -d'=' -f2- | tr -d '"\r') + _OVMS_MODEL_NAME_FILE=$(read_env_value "OVMS_MODEL_NAME" "${ENV_FILE}") +fi +# Read OVMS_MODEL_NAME: shell env var takes precedence over app .env +OVMS_MODEL_NAME_ENV="${OVMS_MODEL_NAME:-${_OVMS_MODEL_NAME_FILE:-Qwen/Qwen2.5-VL-7B-Instruct}}" + +if [ -z "${SUPPORTED_MODEL_SOURCES[${OVMS_MODEL_NAME_ENV}]+x}" ]; then + echo "ERROR: Unsupported OVMS_MODEL_NAME='${OVMS_MODEL_NAME_ENV}'" + echo "Supported models:" + for supported_model in "${!SUPPORTED_MODEL_SOURCES[@]}"; do + echo " - ${supported_model}" + done + echo "Set OVMS_MODEL_NAME in ${ENV_FILE} to one of the values above." + exit 1 +fi + +declare -A MODEL_SOURCES +MODEL_SOURCES["${OVMS_MODEL_NAME_ENV}"]="${SUPPORTED_MODEL_SOURCES[${OVMS_MODEL_NAME_ENV}]}" + +# Optional Hugging Face token loading from the selected app .env. +# If present, export both names for broad compatibility: +# - HF_TOKEN +# - HUGGINGFACE_HUB_TOKEN +_HF_TOKEN_FILE="" +_HUGGINGFACE_HUB_TOKEN_FILE="" +if [ -f "${ENV_FILE}" ]; then + _HF_TOKEN_FILE=$(read_env_value "HF_TOKEN" "${ENV_FILE}") + _HUGGINGFACE_HUB_TOKEN_FILE=$(read_env_value "HUGGINGFACE_HUB_TOKEN" "${ENV_FILE}") +fi + +HF_TOKEN_ENV="${HF_TOKEN:-${HUGGINGFACE_HUB_TOKEN:-${_HF_TOKEN_FILE:-${_HUGGINGFACE_HUB_TOKEN_FILE:-}}}}" +if [ -n "${HF_TOKEN_ENV}" ]; then + export HF_TOKEN="${HF_TOKEN_ENV}" + export HUGGINGFACE_HUB_TOKEN="${HF_TOKEN_ENV}" + echo "Hugging Face authentication: token loaded from environment/.env" +else + echo "Hugging Face authentication: no token found; using anonymous access" fi -# Fall back to the hard-coded source model if .env is missing or unset -OVMS_MODEL_NAME_ENV="${OVMS_MODEL_NAME_ENV:-Qwen/Qwen2.5-VL-7B-Instruct}" # Read TARGET_DEVICE: shell env var takes precedence over take-away/.env, default GPU -_TARGET_DEVICE_FILE=$(grep -E '^TARGET_DEVICE=' "${ENV_FILE}" 2>/dev/null | head -1 | cut -d'=' -f2- | tr -d '"\r') +_TARGET_DEVICE_FILE=$(read_env_value "TARGET_DEVICE" "${ENV_FILE}") TARGET_DEVICE_ENV="${TARGET_DEVICE:-${_TARGET_DEVICE_FILE:-GPU}}" # Read VLM_PRECISION: shell env var takes precedence over take-away/.env, default int8 -_VLM_PRECISION_FILE=$(grep -E '^VLM_PRECISION=' "${ENV_FILE}" 2>/dev/null | head -1 | cut -d'=' -f2- | tr -d '"\r') +_VLM_PRECISION_FILE=$(read_env_value "VLM_PRECISION" "${ENV_FILE}") VLM_PRECISION_ENV="${VLM_PRECISION:-${_VLM_PRECISION_FILE:-int8}}" # Read CACHE_SIZE: KV cache size in GB for OVMS. Default is 4 GB which is @@ -70,7 +110,7 @@ VLM_PRECISION_ENV="${VLM_PRECISION:-${_VLM_PRECISION_FILE:-int8}}" # consumes ~8 GB, so values above 8 will overflow to system RAM and can cause # OOM on 32 GB platforms (ITEP-91499). Users with more VRAM/RAM can raise # this via `export CACHE_SIZE=8` before running setup_models.sh. -_CACHE_SIZE_FILE=$(grep -E '^CACHE_SIZE=' "${ENV_FILE}" 2>/dev/null | head -1 | cut -d'=' -f2- | tr -d '"\r') +_CACHE_SIZE_FILE=$(read_env_value "CACHE_SIZE" "${ENV_FILE}") CACHE_SIZE_ENV="${CACHE_SIZE:-${_CACHE_SIZE_FILE:-4}}" # Validate CACHE_SIZE_ENV is a non-negative integer (0 = dynamic allocation) if ! echo "${CACHE_SIZE_ENV}" | grep -qE '^[0-9]+$'; then @@ -314,18 +354,38 @@ export_model() { target_device_args=(--target_device "${TARGET_DEVICE_ENV}") fi - python "${SCRIPT_DIR}/export_model.py" text_generation \ - --source_model "${SOURCE_MODEL}" \ - --weight-format "${VLM_PRECISION_ENV}" \ - --pipeline_type VLM_CB \ - "${target_device_args[@]}" \ - --cache_size "${CACHE_SIZE_ENV}" \ - --max_num_seqs 4 \ - --max_num_batched_tokens 8192 \ - --enable_prefix_caching True \ - --config_file_path "${MODELS_DIR}/config.json" \ - --model_repository_path "${MODELS_DIR}" \ - --model_name "${MODEL_NAME}" + local export_log + export_log=$(mktemp) + + if ! python "${SCRIPT_DIR}/export_model.py" text_generation \ + --source_model "${SOURCE_MODEL}" \ + --weight-format "${VLM_PRECISION_ENV}" \ + --pipeline_type VLM_CB \ + "${target_device_args[@]}" \ + --cache_size "${CACHE_SIZE_ENV}" \ + --max_num_seqs 4 \ + --max_num_batched_tokens 8192 \ + --enable_prefix_caching True \ + --config_file_path "${MODELS_DIR}/config.json" \ + --model_repository_path "${MODELS_DIR}" \ + --model_name "${MODEL_NAME}" 2>&1 | tee "${export_log}"; then + + if grep -qiE "gated repo|access to model .* is restricted|401 client error|please log in" "${export_log}"; then + echo "" + echo "ERROR: Model download/export requires Hugging Face authentication." + echo "The selected model may be gated." + echo "" + echo "Fix options:" + echo " 1) Add HF_TOKEN= (or HUGGINGFACE_HUB_TOKEN=) to ${ENV_FILE}" + echo " 2) Or run: huggingface-cli login" + echo "" + fi + + rm -f "${export_log}" + return 1 + fi + + rm -f "${export_log}" } ############################################### diff --git a/take-away/.env.example b/take-away/.env.example index 4f5c27eb..863ada9d 100644 --- a/take-away/.env.example +++ b/take-away/.env.example @@ -33,6 +33,11 @@ VLM_PRECISION=int8 TARGET_DEVICE=GPU OVMS_TIMEOUT=120 +# Optional Hugging Face authentication for gated models. +# Leave empty for public models (for example Qwen/Qwen2.5-VL-7B-Instruct). +# setup_models.sh also accepts HUGGINGFACE_HUB_TOKEN as an alternative name. +HF_TOKEN= + # OpenVINO local settings (when VLM_BACKEND=openvino) VLM_MODEL_PATH=/model/Qwen2.5-VL-7B-Instruct From 58da519c2e00f22b6ac872c4c5b36da3991038e7 Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Thu, 23 Jul 2026 11:38:55 +0530 Subject: [PATCH 04/10] VLM prompt engineering, warmup, and benchmark improvements for dine-in vlm_client.py / api.py / validation_service.py / worker.py: - Externalize all VLM prompt templates into configs/prompt_config.json (new) so prompt wording/rules can be tuned without code changes or a container rebuild (file is volume-mounted read-only). Falls back to a built-in default config if the file is missing/invalid. - Add a quantity-counting rule (order servings, not piece count) to fix VLMs over-counting individual pieces (e.g. nuggets) as quantity. - Add VLMClient.warmup(), wired into api.py startup_event(), sending a dummy inference at container boot to absorb the OVMS lazy graph/kernel init penalty (~1-3.5s) before the first real user request. - Add guided/structured decoding via OpenAI-compatible response_format JSON schema (toggle: VLM_GUIDED_DECODING), producing clean minified JSON and reducing completion tokens. - Cap max_tokens via configs/prompt_config.json (default 96) instead of a fixed 512, reducing latency for the compact JSON responses this pipeline actually needs. - Add prediction_debug_logger.py (new): dedicated JSONL debug log (results/logs/vlm_predictions.log) capturing prompt, raw response, parsed output, and performance metadata per request for troubleshooting. - Thread image_filename through worker.py -> api.py -> validation_service -> vlm_client for richer debug logging. configs/orders.json: - Fix MCD-1003/MCD-1004 order manifests, which had each other's expected items swapped, causing false "detection failures" that were actually a test-data bug, not a model issue. Makefile: - benchmark: clean up stale containers (metrics-collector name conflicts) before each run instead of failing silently with all-zero results. - Default BENCHMARK_ITERATIONS 1 -> 8 so a benchmark run exercises all 4 sample images (2 cycles) instead of only the first one. ovms-service/setup_models.sh: - Register OpenVINO/Phi-3.5-vision-instruct-int8-ov and openbmb/MiniCPM-V-2_6 as supported model sources. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 4 + dine-in/Makefile | 5 +- dine-in/configs/orders.json | 8 +- dine-in/configs/prompt_config.json | 22 ++ dine-in/src/api.py | 13 +- .../src/services/prediction_debug_logger.py | 46 +++ dine-in/src/services/validation_service.py | 34 +- dine-in/src/services/vlm_client.py | 373 +++++++++++++++--- dine-in/src/worker.py | 3 +- ovms-service/setup_models.sh | 2 + performance-tools | 2 +- 11 files changed, 452 insertions(+), 60 deletions(-) create mode 100644 dine-in/configs/prompt_config.json create mode 100644 dine-in/src/services/prediction_debug_logger.py diff --git a/.gitignore b/.gitignore index fff2b71d..9c17174f 100755 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,10 @@ model # Benchmark results results/ +dine-in/results_model/ + +# Copilot CLI skills (session-local, not part of the application) +.github/skills/ # Python __pycache__/ diff --git a/dine-in/Makefile b/dine-in/Makefile index 8d2011cd..a3d49364 100755 --- a/dine-in/Makefile +++ b/dine-in/Makefile @@ -83,7 +83,7 @@ REGISTRY_BENCHMARK ?= intel/retail-benchmark:$(TAG) # Benchmark Order Accuracy settings BENCHMARK_WORKERS ?= 1 -BENCHMARK_ITERATIONS ?= 1 +BENCHMARK_ITERATIONS ?= 8 BENCHMARK_DURATION ?= 180 BENCHMARK_TARGET_FPS ?= 15.0 TARGET_DEVICE ?= GPU @@ -440,6 +440,9 @@ benchmark: check-device @echo "Target Device: $(TARGET_DEVICE)" @echo "Results Dir: $(RESULTS_DIR)" @echo "" + @echo "Cleaning up any stale containers from previous runs..." + @docker rm -f metrics-collector dinein_app dinein_ovms_vlm dinein_semantic_service >/dev/null 2>&1 || true + @docker compose -f $(COMPOSE_FILE) --profile benchmark down --remove-orphans >/dev/null 2>&1 || true mkdir -p $(RESULTS_DIR) cd $(PERF_TOOLS_DIR) && \ ( \ diff --git a/dine-in/configs/orders.json b/dine-in/configs/orders.json index 201324f9..223cbb7d 100755 --- a/dine-in/configs/orders.json +++ b/dine-in/configs/orders.json @@ -26,8 +26,8 @@ "restaurant": "McDonald's", "table_number": "T25", "items_ordered": [ - { "item": "Filet-O-Fish", "quantity": 1 }, - { "item": "Cheesy Fries", "quantity": 1 } + { "item": "Chicken Nuggets", "quantity": 1 }, + { "item": "McChicken", "quantity": 1 } ], "order_id": "MCD-1003" }, @@ -36,8 +36,8 @@ "restaurant": "McDonald's", "table_number": "T31", "items_ordered": [ - { "item": "Chicken Nuggets", "quantity": 1 }, - { "item": "McChicken", "quantity": 1 } + { "item": "Cheeseburger", "quantity": 1 }, + { "item": "French Fries", "quantity": 1 } ], "order_id": "MCD-1004" } diff --git a/dine-in/configs/prompt_config.json b/dine-in/configs/prompt_config.json new file mode 100644 index 00000000..79b67f9d --- /dev/null +++ b/dine-in/configs/prompt_config.json @@ -0,0 +1,22 @@ +{ + "$schema_version": "1.0", + "_comment": "VLM prompt templates for food item detection. Edit this file to tune wording/rules without touching code or rebuilding the container. Templates support the placeholder {inventory_list} (Phi variant, newline list) or {inventory_text} (generic variant, comma-separated) which is substituted with the live contents of inventory.json at request time. Restart the dine-in container to pick up changes (no rebuild required — this file is read at runtime).", + + "max_output_tokens": 96, + + "quantity_rule": "\"quantity\" = number of separate ORDER SERVINGS (boxes/wrappers), NEVER the piece count inside one serving. Do NOT count individual nuggets, fries, or pieces. Example: a box containing 6 nuggets is quantity 1, not 6. Only increase quantity if you see 2+ separate boxes/wrappers of the same item.", + + "quantity_field_description": "Number of separate order servings (boxes/wrappers), NOT the piece count inside one serving. One box of nuggets or fries = 1, regardless of how many pieces are visible inside it.", + + "json_schema_example": "{\"items\":[{\"name\":\"item\",\"quantity\":1}]}", + + "phi": { + "with_inventory_template": "You are a food item detector for a restaurant tray.\n\nInventory (the ONLY item names you may use):\n{inventory_list}\n\nTask: Identify which of the inventory items above are present in the image.\n\nReturn ONLY valid JSON using exactly this schema:\n{json_schema_example}\n\nRules:\n- Only detect items from the inventory list above. Never invent names outside this list.\n- Match each visible food item to the closest inventory item name and use that exact name.\n- Carefully scan the entire image before answering.\n- Use all visual evidence in the scene before deciding items.\n- Read visible text on wrappers, cartons, drink cups, and packaging.\n- If product names are visible on packaging, use those names to match an inventory item.\n- Do not rely only on food appearance.\n- Detect every visible food item before generating the response.\n- Do not stop after finding the first item.\n- Include partially visible food items when reasonably confident.\n- Ignore trays, napkins, and background objects.\n- Detect only food items clearly visible in the image.\n- {quantity_rule}\n- Return only valid JSON.\n- Do not output explanations.\n- Do not output reasoning.\n- Never repeat or explain the prompt.\n- Never include markdown.\n- If no inventory items are detected, return exactly: {{\"items\":[]}}" + }, + + "generic": { + "with_inventory_template": "Inventory (the ONLY item names you may use): {inventory_text}\n\nIdentify which of the inventory items above are CLEARLY VISIBLE in this image.\nUse only names from the inventory list. Do NOT guess or include items you cannot see.\n{quantity_rule}\nRespond with MINIFIED JSON on a single line only (no spaces, no newlines, no markdown).\nJSON schema: {json_schema_example}", + + "without_inventory_template": "ONLY list food items CLEARLY VISIBLE in this image. Do NOT guess.\n{quantity_rule}\nRespond with MINIFIED JSON on a single line only (no spaces, no newlines, no markdown).\nJSON: {json_schema_example}" + } +} diff --git a/dine-in/src/api.py b/dine-in/src/api.py index 0f5d3895..5990dcde 100644 --- a/dine-in/src/api.py +++ b/dine-in/src/api.py @@ -480,7 +480,8 @@ async def validate_plate( image_bytes=image_bytes, order_manifest=order_manifest.model_dump(), image_id=image_id, - request_id=request_id + request_id=request_id, + image_filename=image.filename ) inference_end_iso = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f") @@ -600,7 +601,8 @@ async def validate_batch( result = await validation_service.validate_plate( image_bytes=image_bytes, order_manifest=order_manifest.model_dump(), - image_id=image_id + image_id=image_id, + image_filename=image.filename ) # Build response @@ -814,6 +816,13 @@ async def startup_event(): # Check VLM service health await _check_vlm_health() + # Warm up the VLM so the first real request avoids the lazy-init penalty + try: + validation_service = get_validation_service() + await validation_service.vlm_client.warmup() + except Exception as e: + logger.warning(f"[STARTUP] VLM warmup skipped: {e}") + async def _check_vlm_health(): """Check if VLM service is ready and responsive.""" diff --git a/dine-in/src/services/prediction_debug_logger.py b/dine-in/src/services/prediction_debug_logger.py new file mode 100644 index 00000000..8b19cb3a --- /dev/null +++ b/dine-in/src/services/prediction_debug_logger.py @@ -0,0 +1,46 @@ +"""Dedicated JSONL debug logging for VLM predictions and validation results.""" + +import json +import logging +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict + +logger = logging.getLogger(__name__) + + +def _base_dir() -> Path: + return Path(__file__).resolve().parent.parent.parent + + +def _log_path() -> Path: + results_dir = Path(os.getenv("CONTAINER_RESULTS_PATH") or (_base_dir() / "results")) + return results_dir / "logs" / "vlm_predictions.log" + + +def _json_safe(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): _json_safe(val) for key, val in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + if isinstance(value, Path): + return str(value) + return value + + +def write_prediction_debug(record: Dict[str, Any]) -> None: + """Append one structured debug record to the persistent JSONL log.""" + try: + log_path = _log_path() + log_path.parent.mkdir(parents=True, exist_ok=True) + + payload = { + "timestamp": datetime.now(timezone.utc).isoformat(), + **_json_safe(record), + } + + with open(log_path, "a", encoding="utf-8") as handle: + handle.write(json.dumps(payload, ensure_ascii=True) + "\n") + except Exception as exc: + logger.warning(f"Failed to write prediction debug log: {exc}") \ No newline at end of file diff --git a/dine-in/src/services/validation_service.py b/dine-in/src/services/validation_service.py index fba9655a..76dbbd3b 100755 --- a/dine-in/src/services/validation_service.py +++ b/dine-in/src/services/validation_service.py @@ -10,6 +10,7 @@ from .vlm_client import VLMClient, VLMResponse from .semantic_client import SemanticClient, SemanticMatchResult +from .prediction_debug_logger import write_prediction_debug logger = logging.getLogger(__name__) @@ -232,7 +233,8 @@ async def validate_plate( image_bytes: bytes, order_manifest: Dict[str, Any], image_id: str, - request_id: str = None + request_id: str = None, + image_filename: str = None ) -> ValidationResult: """ Validate plate against order manifest using VLM and semantic matching. @@ -252,7 +254,12 @@ async def validate_plate( try: # Step 1: VLM Inference vlm_start = time.time() - vlm_response: VLMResponse = await self.vlm_client.analyze_plate(image_bytes, request_id=request_id) + vlm_response: VLMResponse = await self.vlm_client.analyze_plate( + image_bytes, + request_id=request_id, + image_id=image_id, + image_filename=image_filename + ) vlm_time_ms = (time.time() - vlm_start) * 1000 logger.info(f"VLM inference completed in {vlm_time_ms:.2f}ms for {request_id}, " f"detected {len(vlm_response.detected_items)} items") @@ -332,6 +339,29 @@ async def validate_plate( matched_items=matched_items_list, metrics=metrics ) + + debug_metadata = getattr(vlm_response, 'debug_metadata', {}) or {} + write_prediction_debug({ + "stage": "validation_result", + "request_id": request_id, + "model_name": debug_metadata.get("model_name"), + "image_id": image_id, + "image_filename": image_filename or debug_metadata.get("image_filename"), + "prompt": debug_metadata.get("prompt"), + "raw_response": debug_metadata.get("raw_response"), + "raw_text": vlm_response.raw_content, + "parse_mode": vlm_response.parse_mode, + "parsed_output": vlm_response.parsed_output, + "final_detected_items": vlm_response.detected_items, + "expected_food_items": expected_items, + "missing_items": missing_items, + "extra_items": extra_items, + "matched_items": matched_items_list, + "quantity_mismatches": quantity_mismatches, + "accuracy_score": accuracy_score, + "order_complete": order_complete, + "performance_metadata": vlm_perf, + }) logger.info(f"Validation completed: image_id={image_id}, " f"accuracy={accuracy_score:.2f}, " diff --git a/dine-in/src/services/vlm_client.py b/dine-in/src/services/vlm_client.py index 5c2c39b4..9a4c346a 100644 --- a/dine-in/src/services/vlm_client.py +++ b/dine-in/src/services/vlm_client.py @@ -7,6 +7,7 @@ import base64 import json import logging +import os import time import uuid from dataclasses import dataclass @@ -17,6 +18,7 @@ from io import BytesIO import httpx from PIL import Image, ImageOps, ImageEnhance, ImageFilter +from .prediction_debug_logger import write_prediction_debug from vlm_metrics_logger import ( log_start_time, log_end_time, @@ -248,41 +250,37 @@ def preprocess(self, image_bytes: bytes) -> Tuple[bytes, Dict[str, Any]]: def _smart_resize(self, img: Image.Image) -> Tuple[Image.Image, Dict[str, Any]]: """ - Intelligently resize image while preserving aspect ratio. - - Uses high-quality LANCZOS resampling for downscaling. - Only resizes if image exceeds max_size. + Intelligently resize image into a square while preserving aspect ratio. + + Fits the original image inside a max_size x max_size canvas using + high-quality LANCZOS resampling, then pads the remaining area. """ original_width, original_height = img.size - info: Dict[str, Any] = {"resize_applied": False} - - # Check if resize needed - if max(original_width, original_height) <= self.max_size: - info["resize_reason"] = "already_optimal" - return img, info - - # Calculate new dimensions preserving aspect ratio - if original_width > original_height: - new_width = self.max_size - new_height = int(original_height * (self.max_size / original_width)) - else: - new_height = self.max_size - new_width = int(original_width * (self.max_size / original_height)) - - # Ensure minimum dimensions - new_width = max(new_width, self.MIN_SIZE) - new_height = max(new_height, self.MIN_SIZE) - - # Use LANCZOS for high-quality downsampling - img = img.resize((new_width, new_height), Image.Resampling.LANCZOS) - - info.update({ - "resize_applied": True, - "scale_factor": round(new_width / original_width, 3), - "resize_reason": f"exceeded_max_{self.max_size}" - }) - - return img, info + target_size = self.max_size + scale_factor = min(target_size / original_width, target_size / original_height) + + new_width = max(int(round(original_width * scale_factor)), self.MIN_SIZE) + new_height = max(int(round(original_height * scale_factor)), self.MIN_SIZE) + new_width = min(new_width, target_size) + new_height = min(new_height, target_size) + + resized_img = img.resize((new_width, new_height), Image.Resampling.LANCZOS) + + square_img = Image.new('RGB', (target_size, target_size), (255, 255, 255)) + offset_x = (target_size - new_width) // 2 + offset_y = (target_size - new_height) // 2 + square_img.paste(resized_img, (offset_x, offset_y)) + + info: Dict[str, Any] = { + "resize_applied": (new_width, new_height) != (original_width, original_height), + "square_padding_applied": (offset_x > 0 or offset_y > 0), + "scale_factor": round(scale_factor, 3), + "resized_dimensions": (new_width, new_height), + "padding_offsets": (offset_x, offset_y), + "resize_reason": f"square_letterbox_{target_size}" + } + + return square_img, info def _enhance_contrast(self, img: Image.Image) -> Image.Image: """ @@ -320,6 +318,10 @@ def __init__(self, raw_response: Dict[str, Any]): self.raw_response = raw_response self.detected_items: List[Dict[str, Any]] = [] self.performance_metadata: Dict[str, Any] = {} # Set by VLMClient after inference + self.raw_content: str = "" + self.parsed_output: Any = None + self.parse_mode: str = "unparsed" + self.debug_metadata: Dict[str, Any] = {} self._parse_response() def _parse_response(self): @@ -328,6 +330,7 @@ def _parse_response(self): # Extract content from OpenAI-compatible response if "choices" in self.raw_response: content = self.raw_response["choices"][0]["message"]["content"] + self.raw_content = content logger.info(f"[PARSE] VLM content: {content[:500]}") # Log first 500 chars # Strip markdown code blocks if present (```json ... ```) @@ -345,14 +348,18 @@ def _parse_response(self): # Try to parse as JSON first (structured output) try: parsed_content = json.loads(content_stripped) + self.parsed_output = parsed_content logger.info(f"[PARSE] Successfully parsed JSON: {parsed_content}") if isinstance(parsed_content, dict) and "items" in parsed_content: self.detected_items = parsed_content["items"] + self.parse_mode = "json_dict" logger.info(f"[PARSE] Extracted {len(self.detected_items)} items from JSON dict") elif isinstance(parsed_content, list): self.detected_items = parsed_content + self.parse_mode = "json_list" logger.info(f"[PARSE] Extracted {len(self.detected_items)} items from JSON list") else: + self.parse_mode = "unexpected_json" logger.warning(f"[PARSE] Unexpected JSON structure: {parsed_content}") except json.JSONDecodeError as je: logger.info(f"[PARSE] JSON decode failed: {je}, trying to recover truncated JSON") @@ -360,15 +367,21 @@ def _parse_response(self): recovered_items = self._recover_truncated_json(content_stripped) if recovered_items: self.detected_items = recovered_items + self.parsed_output = recovered_items + self.parse_mode = "truncated_json_recovery" logger.info(f"[PARSE] Recovered {len(self.detected_items)} items from truncated JSON") else: # Fallback: parse natural language response self._parse_natural_language(content) + self.parsed_output = self.detected_items + self.parse_mode = "natural_language" logger.info(f"Parsed {len(self.detected_items)} items from VLM response") else: + self.parse_mode = "invalid_response" logger.error(f"Unexpected VLM response format: {self.raw_response}") except Exception as e: + self.parse_mode = "parse_error" logger.exception(f"Error parsing VLM response: {e}") def _parse_natural_language(self, content: str): @@ -448,6 +461,45 @@ class VLMClient: - Circuit breaker for fault tolerance - Configurable timeouts per operation stage """ + + # Cap output tokens: a full multi-item order response is ~40 tokens of + # minified JSON, so 96 is a safe ceiling that prevents runaway generation + # while keeping latency low (latency scales with completion tokens). + MAX_OUTPUT_TOKENS = 96 + + # JSON schema for guided (structured) decoding. When OVMS supports it, + # this forces valid minified JSON, eliminates markdown fences, and reduces + # completion tokens. Sent via the OpenAI-compatible `response_format` field. + RESPONSE_FORMAT = { + "type": "json_schema", + "json_schema": { + "name": "detected_items", + "strict": True, + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "quantity": { + "type": "integer", + "minimum": 1, + "description": "Number of separate order servings (boxes/wrappers), NOT the piece count inside one serving. One box of nuggets or fries = 1, regardless of how many pieces are visible inside it.", + }, + }, + "required": ["name", "quantity"], + "additionalProperties": False, + }, + } + }, + "required": ["items"], + "additionalProperties": False, + }, + }, + } # Class-level HTTP client pool (shared across instances) _http_client: Optional[httpx.AsyncClient] = None @@ -459,6 +511,22 @@ def __init__(self, endpoint: str, model_name: str, timeout: int = 60): self.timeout = timeout self.chat_endpoint = f"{endpoint}/v3/chat/completions" self.inventory_items = self._load_inventory() + self.prompt_config = self._load_prompt_config() + + # Instance-level max_tokens/response_format so configs/prompt_config.json + # controls these too (falls back to the class-level defaults above if the + # config doesn't specify them). + self.max_output_tokens = self.prompt_config.get("max_output_tokens", self.MAX_OUTPUT_TOKENS) + self.response_format = json.loads(json.dumps(self.RESPONSE_FORMAT)) # deep copy + quantity_description = self.prompt_config.get("quantity_field_description") + if quantity_description: + self.response_format["json_schema"]["schema"]["properties"]["items"]["items"] \ + ["properties"]["quantity"]["description"] = quantity_description + + # Guided (structured) decoding via response_format JSON schema. + # Enabled by default; set VLM_GUIDED_DECODING=false to disable if the + # OVMS build/model does not support it. + self.use_guided_decoding = os.getenv("VLM_GUIDED_DECODING", "true").lower() == "true" # Circuit breaker for OVMS service self._circuit_breaker = CircuitBreaker( @@ -470,9 +538,10 @@ def __init__(self, endpoint: str, model_name: str, timeout: int = 60): ) # Initialize image preprocessor for optimized VLM inference - # Balanced settings for 7B model - good quality with reasonable speed + # Temporary square-resolution benchmark setting for Qwen. + # Change only max_size to test 480, 960, or 1440. self.preprocessor = ImagePreprocessor( - max_size=672, # Good quality for 7B model + max_size=480, jpeg_quality=82, # High quality compression enhance_contrast=True, sharpen=True @@ -519,7 +588,55 @@ async def close_http_client(cls): await cls._http_client.aclose() cls._http_client = None logger.info("Closed shared HTTP client") - + + async def warmup(self) -> bool: + """ + Send a tiny dummy inference to trigger lazy graph/kernel init in OVMS + so the first real user request does not pay the ~3.5s warmup penalty. + + Returns True if the warmup request succeeded, False otherwise. + """ + try: + # Small solid-colour image → minimal preprocessing/encode cost + dummy = Image.new("RGB", (64, 64), (200, 200, 200)) + buf = BytesIO() + dummy.save(buf, format="JPEG", quality=70) + encoded = base64.b64encode(buf.getvalue()).decode("utf-8") + image_url = f"data:image/jpeg;base64,{encoded}" + + payload = { + "model": self.model_name, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Respond with {\"items\":[]}"}, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + } + ], + "max_tokens": 8, + "temperature": 0.0, + } + if self.use_guided_decoding: + payload["response_format"] = self.response_format + + logger.info("[VLM] Warmup inference starting...") + t0 = time.time() + client = await self.get_http_client() + response = await client.post( + self.chat_endpoint, + json=payload, + headers={"Content-Type": "application/json"}, + ) + response.raise_for_status() + logger.info(f"[VLM] Warmup completed in {(time.time() - t0):.2f}s") + return True + except Exception as e: + # Warmup is best-effort: never block startup on failure + logger.warning(f"[VLM] Warmup failed (non-fatal): {e}") + return False + def _load_inventory(self) -> List[str]: """Load inventory items from inventory.json""" try: @@ -539,6 +656,105 @@ def _load_inventory(self) -> List[str]: except Exception as e: logger.error(f"Error loading inventory: {e}") return [] + + # Fallback prompt config used only if configs/prompt_config.json is missing or + # invalid, so the service still starts and behaves predictably. The file is the + # source of truth for prompt wording in normal operation. + _DEFAULT_PROMPT_CONFIG = { + "max_output_tokens": 96, + "quantity_rule": ( + "\"quantity\" = number of separate ORDER SERVINGS (boxes/wrappers), NEVER " + "the piece count inside one serving. Do NOT count individual nuggets, fries, " + "or pieces. Example: a box containing 6 nuggets is quantity 1, not 6. Only " + "increase quantity if you see 2+ separate boxes/wrappers of the same item." + ), + "quantity_field_description": ( + "Number of separate order servings (boxes/wrappers), NOT the piece count " + "inside one serving. One box of nuggets or fries = 1, regardless of how " + "many pieces are visible inside it." + ), + "json_schema_example": '{"items":[{"name":"item","quantity":1}]}', + "phi": { + "with_inventory_template": ( + "You are a food item detector for a restaurant tray.\n\n" + "Inventory (the ONLY item names you may use):\n{inventory_list}\n\n" + "Task: Identify which of the inventory items above are present in the image.\n\n" + "Return ONLY valid JSON using exactly this schema:\n{json_schema_example}\n\n" + "Rules:\n" + "- Only detect items from the inventory list above. Never invent names outside this list.\n" + "- Match each visible food item to the closest inventory item name and use that exact name.\n" + "- Carefully scan the entire image before answering.\n" + "- Use all visual evidence in the scene before deciding items.\n" + "- Read visible text on wrappers, cartons, drink cups, and packaging.\n" + "- If product names are visible on packaging, use those names to match an inventory item.\n" + "- Do not rely only on food appearance.\n" + "- Detect every visible food item before generating the response.\n" + "- Do not stop after finding the first item.\n" + "- Include partially visible food items when reasonably confident.\n" + "- Ignore trays, napkins, and background objects.\n" + "- Detect only food items clearly visible in the image.\n" + "- {quantity_rule}\n" + "- Return only valid JSON.\n" + "- Do not output explanations.\n" + "- Do not output reasoning.\n" + "- Never repeat or explain the prompt.\n" + "- Never include markdown.\n" + "- If no inventory items are detected, return exactly: {{\"items\":[]}}" + ), + }, + "generic": { + "with_inventory_template": ( + "Inventory (the ONLY item names you may use): {inventory_text}\n\n" + "Identify which of the inventory items above are CLEARLY VISIBLE in this image.\n" + "Use only names from the inventory list. Do NOT guess or include items you cannot see.\n" + "{quantity_rule}\n" + "Respond with MINIFIED JSON on a single line only (no spaces, no newlines, no markdown).\n" + "JSON schema: {json_schema_example}" + ), + "without_inventory_template": ( + "ONLY list food items CLEARLY VISIBLE in this image. Do NOT guess.\n" + "{quantity_rule}\n" + "Respond with MINIFIED JSON on a single line only (no spaces, no newlines, no markdown).\n" + "JSON: {json_schema_example}" + ), + }, + } + + def _load_prompt_config(self) -> Dict[str, Any]: + """ + Load prompt templates/wording from configs/prompt_config.json so prompt + engineering can be tuned without touching code or rebuilding the image. + Falls back to _DEFAULT_PROMPT_CONFIG (and logs a warning) if the file is + missing or invalid, so a bad edit can never take the service down. + """ + config = json.loads(json.dumps(self._DEFAULT_PROMPT_CONFIG)) # deep copy + try: + base_dir = Path(__file__).resolve().parent.parent.parent + prompt_config_path = base_dir / "configs" / "prompt_config.json" + + if not prompt_config_path.exists(): + logger.warning( + f"Prompt config not found at {prompt_config_path}, using built-in defaults" + ) + return config + + with open(prompt_config_path, 'r') as f: + loaded = json.load(f) + + # Shallow-merge top level, deep-merge the "phi"/"generic" template groups + for key, value in loaded.items(): + if key.startswith("_") or key.startswith("$"): + continue # skip documentation/metadata keys + if key in ("phi", "generic") and isinstance(value, dict): + config.setdefault(key, {}).update(value) + else: + config[key] = value + + logger.info(f"Loaded prompt config from {prompt_config_path}") + return config + except Exception as e: + logger.error(f"Error loading prompt config, using built-in defaults: {e}") + return config def _encode_image(self, image_bytes: bytes, skip_preprocessing: bool = False) -> Tuple[str, Dict[str, Any]]: """ @@ -575,18 +791,41 @@ def _encode_image(self, image_bytes: bytes, skip_preprocessing: bool = False) -> raise def _build_prompt(self) -> str: - """Build ultra-compact prompt for fast inference on iGPU""" - if self.inventory_items: - # Compact format: comma-separated items (faster than numbered list) - inventory_text = ", ".join(self.inventory_items[:30]) # Limit to 30 items - prompt = f"""Known menu items: {inventory_text} + """ + Build the VLM prompt from configs/prompt_config.json (loaded at init into + self.prompt_config). All wording/rules live in that config file — edit it + and restart the container to change prompt behavior; no code change needed. + """ + cfg = self.prompt_config + quantity_rule = cfg.get("quantity_rule", "") + json_schema_example = cfg.get("json_schema_example", '{"items":[{"name":"item","quantity":1}]}') + + if self.model_name.startswith("OpenVINO/Phi"): + inventory_list = ", ".join(self.inventory_items) if self.inventory_items else "" + template = cfg.get("phi", {}).get("with_inventory_template", "") + prompt = template.format( + inventory_list=inventory_list, + quantity_rule=quantity_rule, + json_schema_example=json_schema_example, + ) + logger.info(f"[PROMPT] Built Phi-specific strict JSON prompt with {len(self.inventory_items)} inventory items, length={len(prompt)} chars") + return prompt -ONLY list food items CLEARLY VISIBLE in this image. Do NOT guess or include items you cannot see. -Return JSON: {{"items":[{{"name":"item","quantity":1}}]}}""" + if self.inventory_items: + inventory_text = ", ".join(self.inventory_items) + template = cfg.get("generic", {}).get("with_inventory_template", "") + prompt = template.format( + inventory_text=inventory_text, + quantity_rule=quantity_rule, + json_schema_example=json_schema_example, + ) else: - prompt = """ONLY list food items CLEARLY VISIBLE in this image. Do NOT guess. -JSON: {"items":[{"name":"item","quantity":1}]}""" - + template = cfg.get("generic", {}).get("without_inventory_template", "") + prompt = template.format( + quantity_rule=quantity_rule, + json_schema_example=json_schema_example, + ) + logger.info(f"[PROMPT] Built compact prompt with {len(self.inventory_items)} inventory items, length={len(prompt)} chars") return prompt @@ -594,7 +833,9 @@ async def analyze_plate( self, image_bytes: bytes, order_id: Optional[str] = None, - request_id: Optional[str] = None + request_id: Optional[str] = None, + image_id: Optional[str] = None, + image_filename: Optional[str] = None ) -> VLMResponse: """ Analyze food plate image using VLM with optimized preprocessing. @@ -640,20 +881,25 @@ async def analyze_plate( f"dims={preprocess_meta.get('processed_dimensions', 'N/A')}") # Step 2: Build request payload (OpenAI-compatible format) + prompt_text = self._build_prompt() payload = { "model": self.model_name, "messages": [ { "role": "user", "content": [ - {"type": "text", "text": self._build_prompt()}, + {"type": "text", "text": prompt_text}, {"type": "image_url", "image_url": {"url": encoded_image}} ] } ], - "max_tokens": 512, # Increased to handle longer item lists + "max_tokens": self.max_output_tokens, # Cap output; response is compact JSON "temperature": 0.0 # Greedy decoding for fastest inference } + + # Enable structured decoding to guarantee valid minified JSON + if self.use_guided_decoding: + payload["response_format"] = self.response_format # Step 3: Check circuit breaker before making request if not await self._circuit_breaker.can_execute(): @@ -743,6 +989,20 @@ async def analyze_plate( # Step 5: Create response and parse detected items vlm_response = VLMResponse(result) + vlm_response.debug_metadata = { + "request_id": req_id, + "model_name": self.model_name, + "image_id": image_id, + "image_filename": image_filename, + "prompt": prompt_text, + "raw_response": result, + "raw_text": text, + "payload_settings": { + "max_tokens": payload["max_tokens"], + "temperature": payload["temperature"] + }, + "preprocess_metadata": preprocess_meta, + } # Attach performance metadata to response vlm_response.performance_metadata = { @@ -759,6 +1019,21 @@ async def analyze_plate( "tpot_sec": round(tpot, 4), "throughput_mean_sec": round(throughput_mean, 2) } + + write_prediction_debug({ + "stage": "vlm_inference", + "request_id": req_id, + "model_name": self.model_name, + "image_id": image_id, + "image_filename": image_filename, + "prompt": prompt_text, + "raw_response": result, + "raw_text": text, + "parse_mode": vlm_response.parse_mode, + "parsed_output": vlm_response.parsed_output, + "detected_items": vlm_response.detected_items, + "performance_metadata": vlm_response.performance_metadata, + }) # Log custom metrics event log_custom_event( diff --git a/dine-in/src/worker.py b/dine-in/src/worker.py index e884af46..01d7cba8 100644 --- a/dine-in/src/worker.py +++ b/dine-in/src/worker.py @@ -272,7 +272,8 @@ async def process_image(self, image_path: Path, iteration: int) -> WorkerResult: image_bytes=image_bytes, order_manifest={"items": order_data.get("items", [])}, image_id=image_id, - request_id=request_id + request_id=request_id, + image_filename=image_path.name ) # Extract metrics diff --git a/ovms-service/setup_models.sh b/ovms-service/setup_models.sh index 18313e51..4e0194ec 100755 --- a/ovms-service/setup_models.sh +++ b/ovms-service/setup_models.sh @@ -39,6 +39,8 @@ fi declare -A SUPPORTED_MODEL_SOURCES SUPPORTED_MODEL_SOURCES["Qwen/Qwen2.5-VL-7B-Instruct"]="Qwen/Qwen2.5-VL-7B-Instruct" SUPPORTED_MODEL_SOURCES["openbmb/MiniCPM-V-4_5"]="openbmb/MiniCPM-V-4_5" +SUPPORTED_MODEL_SOURCES["openbmb/MiniCPM-V-2_6"]="openbmb/MiniCPM-V-2_6" +SUPPORTED_MODEL_SOURCES["OpenVINO/Phi-3.5-vision-instruct-int8-ov"]="OpenVINO/Phi-3.5-vision-instruct-int8-ov" POTENTIAL_SOURCE_DIRS=( "${HOME}/ovms-vlm/models" diff --git a/performance-tools b/performance-tools index 8e5a847e..cd373f44 160000 --- a/performance-tools +++ b/performance-tools @@ -1 +1 @@ -Subproject commit 8e5a847e29998c73f003e9412af74f80d1b3c3e5 +Subproject commit cd373f445e14465dde1f96a36fdd50a37fe7341e From bfeb2d8606bf79696042dbd89799240e8b7b0bba Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Tue, 28 Jul 2026 10:55:48 +0530 Subject: [PATCH 05/10] Benchmark MiniCPM-V-4_5 INT4 at 448px resolution for dine-in order accuracy Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dine-in/src/services/vlm_client.py | 2 +- ovms-service/setup_models.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dine-in/src/services/vlm_client.py b/dine-in/src/services/vlm_client.py index 9a4c346a..b1f3ce52 100644 --- a/dine-in/src/services/vlm_client.py +++ b/dine-in/src/services/vlm_client.py @@ -541,7 +541,7 @@ def __init__(self, endpoint: str, model_name: str, timeout: int = 60): # Temporary square-resolution benchmark setting for Qwen. # Change only max_size to test 480, 960, or 1440. self.preprocessor = ImagePreprocessor( - max_size=480, + max_size=448, jpeg_quality=82, # High quality compression enhance_contrast=True, sharpen=True diff --git a/ovms-service/setup_models.sh b/ovms-service/setup_models.sh index 4e0194ec..3db4bdb6 100755 --- a/ovms-service/setup_models.sh +++ b/ovms-service/setup_models.sh @@ -39,6 +39,15 @@ fi declare -A SUPPORTED_MODEL_SOURCES SUPPORTED_MODEL_SOURCES["Qwen/Qwen2.5-VL-7B-Instruct"]="Qwen/Qwen2.5-VL-7B-Instruct" SUPPORTED_MODEL_SOURCES["openbmb/MiniCPM-V-4_5"]="openbmb/MiniCPM-V-4_5" +# NOTE: "openbmb/MiniCPM-V-4_5-int4" on Hugging Face is a bitsandbytes-quantized +# PyTorch checkpoint (tagged "8-bit"), which optimum-intel/NNCF cannot import +# directly into OpenVINO IR. The verified, reproducible path is to export from +# the full-precision "openbmb/MiniCPM-V-4_5" source and let NNCF perform its own +# INT4 weight compression (--weight-format int4 / VLM_PRECISION=int4), writing +# the result to a locally-named "-int4" directory to reflect the applied +# precision. This entry documents that convention so setup_models.sh can +# reproduce the model already present under models/openbmb/MiniCPM-V-4_5-int4. +SUPPORTED_MODEL_SOURCES["openbmb/MiniCPM-V-4_5-int4"]="openbmb/MiniCPM-V-4_5" SUPPORTED_MODEL_SOURCES["openbmb/MiniCPM-V-2_6"]="openbmb/MiniCPM-V-2_6" SUPPORTED_MODEL_SOURCES["OpenVINO/Phi-3.5-vision-instruct-int8-ov"]="OpenVINO/Phi-3.5-vision-instruct-int8-ov" From 31ed8aa7410eff8d030b5cdaaef7e20ca626807a Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Tue, 28 Jul 2026 11:39:03 +0530 Subject: [PATCH 06/10] Address Copilot PR #159 review comments - setup_models.sh: use PIPESTATUS[0] instead of tee's exit code so model export failures are correctly detected instead of masked by the pipeline's success. - ovms-service/README.md: list all models actually supported by setup_models.sh (MiniCPM-V-4_5-int4, MiniCPM-V-2_6, Phi-3.5-vision), not just Qwen and MiniCPM-V-4_5. - station_worker.py: tie rtspsrc's internal timeout to the timeout_sec parameter (in microseconds) instead of a hardcoded 5s, so it stays consistent with caller-provided timeouts. - prediction_debug_logger.py: gate JSONL debug logging behind VLM_PREDICTION_DEBUG_LOG env flag (default off) to avoid unbounded disk growth/I O in production. - vlm_client.py: join Phi inventory items with newlines instead of commas to match the documented {inventory_list} placeholder format. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dine-in/src/services/prediction_debug_logger.py | 13 ++++++++++++- dine-in/src/services/vlm_client.py | 2 +- ovms-service/README.md | 6 ++++++ ovms-service/setup_models.sh | 9 +++++++-- take-away/src/parallel/station_worker.py | 2 +- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/dine-in/src/services/prediction_debug_logger.py b/dine-in/src/services/prediction_debug_logger.py index 8b19cb3a..d55b4191 100644 --- a/dine-in/src/services/prediction_debug_logger.py +++ b/dine-in/src/services/prediction_debug_logger.py @@ -29,8 +29,19 @@ def _json_safe(value: Any) -> Any: return value +def _debug_logging_enabled() -> bool: + return os.getenv("VLM_PREDICTION_DEBUG_LOG", "false").lower() == "true" + + def write_prediction_debug(record: Dict[str, Any]) -> None: - """Append one structured debug record to the persistent JSONL log.""" + """Append one structured debug record to the persistent JSONL log. + + No-op unless VLM_PREDICTION_DEBUG_LOG=true, to avoid unbounded disk + growth and extra I/O in production deployments. + """ + if not _debug_logging_enabled(): + return + try: log_path = _log_path() log_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/dine-in/src/services/vlm_client.py b/dine-in/src/services/vlm_client.py index b1f3ce52..21de09b1 100644 --- a/dine-in/src/services/vlm_client.py +++ b/dine-in/src/services/vlm_client.py @@ -801,7 +801,7 @@ def _build_prompt(self) -> str: json_schema_example = cfg.get("json_schema_example", '{"items":[{"name":"item","quantity":1}]}') if self.model_name.startswith("OpenVINO/Phi"): - inventory_list = ", ".join(self.inventory_items) if self.inventory_items else "" + inventory_list = "\n".join(self.inventory_items) if self.inventory_items else "" template = cfg.get("phi", {}).get("with_inventory_template", "") prompt = template.format( inventory_list=inventory_list, diff --git a/ovms-service/README.md b/ovms-service/README.md index f5a45432..c9345f3e 100755 --- a/ovms-service/README.md +++ b/ovms-service/README.md @@ -47,6 +47,12 @@ Supported values for `OVMS_MODEL_NAME`: - `Qwen/Qwen2.5-VL-7B-Instruct` - `openbmb/MiniCPM-V-4_5` +- `openbmb/MiniCPM-V-4_5-int4` +- `openbmb/MiniCPM-V-2_6` +- `OpenVINO/Phi-3.5-vision-instruct-int8-ov` + +> This list mirrors `SUPPORTED_MODEL_SOURCES` in `ovms-service/setup_models.sh`, +> which is the source of truth — check that script if this list appears out of date. Optional authentication for gated Hugging Face models: diff --git a/ovms-service/setup_models.sh b/ovms-service/setup_models.sh index 3db4bdb6..a8f9da43 100755 --- a/ovms-service/setup_models.sh +++ b/ovms-service/setup_models.sh @@ -368,7 +368,7 @@ export_model() { local export_log export_log=$(mktemp) - if ! python "${SCRIPT_DIR}/export_model.py" text_generation \ + python "${SCRIPT_DIR}/export_model.py" text_generation \ --source_model "${SOURCE_MODEL}" \ --weight-format "${VLM_PRECISION_ENV}" \ --pipeline_type VLM_CB \ @@ -379,7 +379,12 @@ export_model() { --enable_prefix_caching True \ --config_file_path "${MODELS_DIR}/config.json" \ --model_repository_path "${MODELS_DIR}" \ - --model_name "${MODEL_NAME}" 2>&1 | tee "${export_log}"; then + --model_name "${MODEL_NAME}" 2>&1 | tee "${export_log}" + # Use PIPESTATUS[0] instead of $? so the export's own exit code (not + # tee's) determines success/failure of the pipeline. + local export_status="${PIPESTATUS[0]}" + + if [ "${export_status}" -ne 0 ]; then if grep -qiE "gated repo|access to model .* is restricted|401 client error|please log in" "${export_log}"; then echo "" diff --git a/take-away/src/parallel/station_worker.py b/take-away/src/parallel/station_worker.py index e9cae5d2..fe9dc904 100755 --- a/take-away/src/parallel/station_worker.py +++ b/take-away/src/parallel/station_worker.py @@ -933,7 +933,7 @@ def _verify_rtsp_stream_quick(self, timeout_sec: int = 3) -> bool: 'timeout', str(timeout_sec), 'gst-launch-1.0', '-e', 'rtspsrc', f'location={self.rtsp_url}', 'protocols=tcp', 'latency=500', - 'timeout=5000000', # 5 second RTSP timeout + f'timeout={timeout_sec * 1000000}', # rtspsrc timeout is in microseconds '!', 'fakesink', 'sync=false' ], capture_output=True, From 0f0990bc5ae1478b26d6bcc868b1780f999faae0 Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Tue, 28 Jul 2026 13:13:46 +0530 Subject: [PATCH 07/10] Validate RTSP_LATENCY/CAPTURE_FPS env vars before shell interpolation Address Copilot review on PR #159: RTSP_LATENCY was read as a raw string and interpolated into a GStreamer pipeline executed with shell=True. Parse both env vars as integers with range checks and safe defaults. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- take-away/src/core/pipeline_runner.py | 28 +++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/take-away/src/core/pipeline_runner.py b/take-away/src/core/pipeline_runner.py index 71217a7c..baabed9b 100755 --- a/take-away/src/core/pipeline_runner.py +++ b/take-away/src/core/pipeline_runner.py @@ -4,8 +4,32 @@ import logging from minio import Minio -RTSP_DEFAULT_LATENCY = os.getenv("RTSP_LATENCY", "500") -CAPTURE_FPS = int(os.getenv("CAPTURE_FPS", "10")) +def _int_env(name: str, default: int, minimum: int = 0) -> int: + """Read an integer env var, falling back to `default` when unset/invalid. + + Values from the environment are interpolated into a shell-executed + GStreamer pipeline string, so they must be strictly validated as integers. + """ + raw = os.getenv(name) + if raw is None: + return default + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + logging.getLogger(__name__).warning( + f"Invalid {name}={raw!r}; falling back to {default}" + ) + return default + if value < minimum: + logging.getLogger(__name__).warning( + f"{name}={value} below minimum {minimum}; falling back to {default}" + ) + return default + return value + + +RTSP_DEFAULT_LATENCY = _int_env("RTSP_LATENCY", 500) +CAPTURE_FPS = _int_env("CAPTURE_FPS", 10, minimum=1) queue_params = "max-size-time=0 max-size-bytes=0 max-size-buffers=200 leaky=no" # Configure logging From d4b377c588b8764d47b0e5b6a6cd2b2a5bb29328 Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Wed, 29 Jul 2026 18:28:22 +0530 Subject: [PATCH 08/10] Fix dine-in redeploy, NVL INT8 segfault, and GPU metrics reporting ITEP-93258: make up aborted on redeploy when the container had chowned the bind-mounted results/ directory to a UID the host user does not own. chmod now tolerates EPERM, matching the idiom already used at Makefile:605. ITEP-94280: NNCF INT8 calibration can abort with SIGSEGV inside the OpenVINO CPU plugin on NVL. Because the download, FP32 export and INT8 quantization shared one heredoc under `set -e`, the crash killed setup entirely and skipped artifact verification. INT8 now runs as an isolated step that traps the exit code, retries once with a conservative ONEDNN_MAX_CPU_ISA, discards partial IR, and degrades to FP32 instead of failing setup. frame_selector falls back to the FP32 model when the INT8 IR is absent. ITEP-91371: gpu_utilization reported 0.0 for a busy GPU. GPU samples are synthesised from qmassa output and cached, and _extract_peak_from_series silently fell back to the last sample (post-inference idle) when no sample fell inside the window, so a dead collector, a stalled collector, a clock skew and a genuinely idle GPU all produced an identical 0.0. Extraction now returns a status alongside the value, never passes an out-of-window sample off as in-window, exposes metrics_status in the API response, and logs an actionable warning. gpu_memory_utilization was hardcoded to 0.0 and is now reported as unsupported. Also tag the dine-in image as :latest. DINEIN_TAG is kept separate from TAG because TAG also selects intel/retail-benchmark:$(TAG), which has no latest tag. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dine-in/Makefile | 9 +- dine-in/docker-compose.yaml | 4 +- dine-in/src/api.py | 128 +++++++++++++++--- ovms-service/setup_models.sh | 107 ++++++++++++--- .../app/frame_selector.py | 14 +- 5 files changed, 213 insertions(+), 49 deletions(-) diff --git a/dine-in/Makefile b/dine-in/Makefile index a3d49364..c2dfc943 100755 --- a/dine-in/Makefile +++ b/dine-in/Makefile @@ -71,7 +71,11 @@ export TAG = 2026.1.0 # Dine-In image configuration DINEIN_IMAGE_NAME ?= intel/order-accuracy-dine-in -DINEIN_IMAGE ?= $(DINEIN_IMAGE_NAME):$(TAG) +# Tag for the dine-in image only. Kept separate from TAG because TAG also +# selects the benchmark image (intel/retail-benchmark:$(TAG)), which is +# published per-version and has no "latest" tag. +DINEIN_TAG ?= latest +DINEIN_IMAGE ?= $(DINEIN_IMAGE_NAME):$(DINEIN_TAG) # Registry configuration # REGISTRY=false : Build and use local images @@ -263,7 +267,8 @@ check-device: ## Check that graph.pbtxt device matches TARGET_DEVICE (auto-updat up: check-device @echo "Starting dine-in services..." - @mkdir -p results && chmod 777 results + @mkdir -p results + @chmod 777 results 2>/dev/null || true @if [ "$(REGISTRY)" = "false" ]; then \ echo "$(BLUE)Using locally built image: $(DINEIN_IMAGE)$(NC)"; \ DINEIN_IMAGE=$(DINEIN_IMAGE) docker compose up -d --no-build; \ diff --git a/dine-in/docker-compose.yaml b/dine-in/docker-compose.yaml index f4ff66a8..1fe4d4c5 100644 --- a/dine-in/docker-compose.yaml +++ b/dine-in/docker-compose.yaml @@ -73,7 +73,7 @@ services: # Dine-In Order Accuracy Gradio UI dine-in: - image: ${DINEIN_IMAGE:-intel/order-accuracy-dine-in:2026.1.0} + image: ${DINEIN_IMAGE:-intel/order-accuracy-dine-in:latest} build: context: . dockerfile: Dockerfile @@ -116,7 +116,7 @@ services: # Scale with: docker compose up -d --scale dinein-worker=4 # Or set WORKERS env variable and use profile: docker compose --profile benchmark up -d dinein-worker: - image: ${DINEIN_IMAGE:-intel/order-accuracy-dine-in:2026.1.0} + image: ${DINEIN_IMAGE:-intel/order-accuracy-dine-in:latest} build: context: . dockerfile: Dockerfile diff --git a/dine-in/src/api.py b/dine-in/src/api.py index 5990dcde..9b248b9c 100644 --- a/dine-in/src/api.py +++ b/dine-in/src/api.py @@ -11,7 +11,7 @@ import time from datetime import datetime, timezone from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from io import BytesIO from collections import OrderedDict @@ -182,24 +182,62 @@ def _extract_peak_from_series( start_iso: str, end_iso: str, value_index: int = 1, -) -> float: + stale_tolerance_s: float = 10.0, +) -> Tuple[Optional[float], str]: """ - Return the peak value from a time-series list within [start_iso, end_iso]. + Return (peak_value, status) from a time-series within [start_iso, end_iso]. - Each entry is [timestamp_str, value, ...]. Falls back to the last entry - if no samples fall inside the window. + Each entry is [timestamp_str, value, ...]. + + Returning a status alongside the value is deliberate: a genuine "0 % measured" + and "the collector gave us nothing" are very different facts, and collapsing + both to 0.0 is what made this metric untrustworthy (ITEP-91371). + + status is one of: + "ok" - at least one sample fell inside the inference window + "approximate" - no in-window sample, but one within *stale_tolerance_s* + of the window was used instead + "no-data" - the series was empty + "stale" - the series exists but is too old to describe this window """ + if not series: + return None, "no-data" + peak = None for entry in series: - ts = entry[0] - if start_iso <= ts <= end_iso: + if start_iso <= entry[0] <= end_iso: val = entry[value_index] if peak is None or val > peak: peak = val if peak is not None: - return peak - # Fallback: return the last value if the window yielded nothing - return series[-1][value_index] if series else 0.0 + return peak, "ok" + + # No sample inside the window. The previous implementation silently returned + # the last sample of the series; for GPU that is the post-inference idle + # reading (or an arbitrarily old cached one), which is how a busy GPU came + # to be reported as 0.0 %. Only accept a nearby sample, and say so. + def _parse(ts: str) -> Optional[datetime]: + try: + return datetime.fromisoformat(ts) + except (ValueError, TypeError): + return None + + win_start, win_end = _parse(start_iso), _parse(end_iso) + if win_start is None or win_end is None: + return None, "stale" + + nearest_val, nearest_gap = None, None + for entry in series: + ts = _parse(entry[0]) + if ts is None: + continue + gap = (win_start - ts).total_seconds() if ts < win_start else (ts - win_end).total_seconds() + if nearest_gap is None or gap < nearest_gap: + nearest_gap, nearest_val = gap, entry[value_index] + + if nearest_val is not None and nearest_gap is not None and nearest_gap <= stale_tolerance_s: + return nearest_val, "approximate" + return None, "stale" async def call_metrics_collector( @@ -235,31 +273,68 @@ async def call_metrics_collector( gpu_series = metrics_data.get('gpu_utilization', []) memory_series = metrics_data.get('memory', []) + statuses: Dict[str, str] = {} + if inference_start_iso and inference_end_iso: # Peak utilization during the inference window - cpu_util = _extract_peak_from_series( + cpu_util, statuses["cpu_utilization"] = _extract_peak_from_series( cpu_series, inference_start_iso, inference_end_iso) - gpu_util = _extract_peak_from_series( + gpu_util, statuses["gpu_utilization"] = _extract_peak_from_series( gpu_series, inference_start_iso, inference_end_iso) # Memory: percent is at index 4 - memory_util = _extract_peak_from_series( - memory_series, inference_start_iso, inference_end_iso, - value_index=4) if memory_series and len(memory_series[0]) > 4 else 0.0 + if memory_series and len(memory_series[0]) > 4: + memory_util, statuses["memory_utilization"] = _extract_peak_from_series( + memory_series, inference_start_iso, inference_end_iso, + value_index=4) + else: + memory_util, statuses["memory_utilization"] = None, "no-data" mode = "peak-in-window" else: # Snapshot: latest values (original behaviour) - cpu_util = cpu_series[-1][1] if cpu_series else 0.0 - gpu_util = gpu_series[-1][1] if gpu_series else 0.0 + cpu_util = cpu_series[-1][1] if cpu_series else None + gpu_util = gpu_series[-1][1] if gpu_series else None memory_util = (memory_series[-1][4] if memory_series and len(memory_series[-1]) > 4 - else 0.0) + else None) + for name, val in (("cpu_utilization", cpu_util), + ("gpu_utilization", gpu_util), + ("memory_utilization", memory_util)): + statuses[name] = "ok" if val is not None else "no-data" mode = "latest-snapshot" + # gpu_memory_utilization is not exposed by the metrics-collector at all. + statuses["gpu_memory_utilization"] = "unsupported" + + # Unavailable metrics are reported as 0.0 to keep the response schema + # stable, but the accompanying status says why, and a warning is + # logged so a broken collector is visible instead of silent. + # "unsupported" is a known permanent gap, not a fault, so it is not warned about. + unavailable = {n: s for n, s in statuses.items() + if s not in ("ok", "approximate", "unsupported")} + if unavailable: + logger.warning( + f"[METRICS] Utilization unavailable for {sorted(unavailable)} " + f"(status={unavailable}); reported as 0.0. " + f"series sizes: cpu={len(cpu_series)} gpu={len(gpu_series)} " + f"memory={len(memory_series)}. " + f"window={inference_start_iso}..{inference_end_iso}. " + f"If the GPU is known to be busy, check that qmassa is running " + f"in the metrics-collector container (/tmp/results/" + f"qmassa1-*-tool-generated.json and qmassa_error.log)." + ) + approximate = [n for n, s in statuses.items() if s == "approximate"] + if approximate: + logger.info( + f"[METRICS] No in-window samples for {sorted(approximate)}; " + f"used the nearest sample outside the window." + ) + metrics_response = { - "cpu_utilization": round(cpu_util, 2), - "gpu_utilization": round(gpu_util, 2), - "memory_utilization": round(memory_util, 2), + "cpu_utilization": round(cpu_util, 2) if cpu_util is not None else 0.0, + "gpu_utilization": round(gpu_util, 2) if gpu_util is not None else 0.0, + "memory_utilization": round(memory_util, 2) if memory_util is not None else 0.0, "gpu_memory_utilization": 0.0, # Not provided by metrics-collector + "metrics_status": statuses, } logger.info(f"[METRICS] System metrics ({mode}): {metrics_response}") @@ -273,6 +348,12 @@ async def call_metrics_collector( "gpu_utilization": 0.0, "memory_utilization": 0.0, "gpu_memory_utilization": 0.0, + "metrics_status": { + "cpu_utilization": "collector-unreachable", + "gpu_utilization": "collector-unreachable", + "memory_utilization": "collector-unreachable", + "gpu_memory_utilization": "collector-unreachable", + }, } @@ -514,7 +595,10 @@ async def validate_plate( "cpu_utilization": system_metrics.get("cpu_utilization", 0.0), "gpu_utilization": system_metrics.get("gpu_utilization", 0.0), "memory_utilization": system_metrics.get("memory_utilization", 0.0), - "gpu_memory_utilization": system_metrics.get("gpu_memory_utilization", 0.0) + "gpu_memory_utilization": system_metrics.get("gpu_memory_utilization", 0.0), + # Per-metric availability, so "0.0" can be told apart from + # "the collector had nothing for this window" (ITEP-91371). + "metrics_status": system_metrics.get("metrics_status", {}), } logger.info(f"[API] Metrics for {request_id}: e2e={end_to_end_ms:.0f}ms, decode={image_decode_ms:.0f}ms, vlm={vlm_latency_ms}ms, semantic={semantic_matching_ms}ms") diff --git a/ovms-service/setup_models.sh b/ovms-service/setup_models.sh index a8f9da43..1bc1c834 100755 --- a/ovms-service/setup_models.sh +++ b/ovms-service/setup_models.sh @@ -661,7 +661,7 @@ else mkdir -p "${YOLO_MODEL_DIR}" "${YOLO_DATASETS_DIR}" - echo "[1/2] Downloading yolo11n.pt and exporting OpenVINO models..." + echo "[1/3] Downloading yolo11n.pt and exporting OpenVINO FP32 model..." YOLO_MODEL_DIR="${YOLO_MODEL_DIR}" YOLO_DATASETS_DIR="${YOLO_DATASETS_DIR}" \ python3 - << 'PYEOF' import os, sys @@ -700,26 +700,89 @@ if not fp32_dir.exists(): else: print(f" FP32 model already exists: {fp32_dir}") -# ── Step 3: INT8 quantization ──────────────────────────────────────────────── -if not int8_dir.exists(): - print(" Quantizing to OpenVINO INT8 (downloads COCO128 ~7 MB if needed) ...") - orig = os.getcwd() - os.chdir(str(model_dir)) - YOLO(str(yolo_pt)).export(format="openvino", int8=True, data="coco128.yaml") - # ultralytics exports INT8 to "yolo11n_openvino_model/" in CWD; - # rename it so it doesn't overwrite the FP32 export. - default_out = Path("yolo11n_openvino_model") - if default_out.exists() and not int8_dir.exists(): - default_out.rename(int8_dir.name) - os.chdir(orig) - print(f" ✓ INT8 quantization: {int8_dir}") -else: - print(f" INT8 model already exists: {int8_dir}") +# ── Step 3 (INT8 quantization) runs as a separate, isolated invocation ─────── +# See the "[2/3]" block in setup_models.sh: NNCF calibration can abort with +# SIGSEGV inside the OpenVINO CPU plugin, which must not kill the whole script. -print("YOLO export complete.") +print("YOLO base + FP32 export complete.") PYEOF - echo "[2/2] Verifying YOLO artifacts..." + # ── INT8 quantization: isolated and non-fatal (ITEP-94280) ─────────── + # NNCF calibration runs real inference through the OpenVINO CPU plugin. + # On CPUs the plugin does not yet recognise, that inference can abort with + # SIGSEGV during statistics collection. Because this script runs under + # `set -e`, an inline crash previously killed setup outright and skipped + # artifact verification. Run it in isolation, retry once with a + # conservative oneDNN ISA, and degrade to FP32 rather than failing setup. + if [ -d "${YOLO_INT8_DIR}" ]; then + echo "[2/3] INT8 model already exists, skipping quantization." + else + echo "[2/3] Quantizing to OpenVINO INT8 (downloads COCO128 ~7 MB if needed)..." + _int8_script="$(mktemp /tmp/yolo_int8_XXXXXX.py)" + cat > "${_int8_script}" << 'PYEOF' +import os +from pathlib import Path +from ultralytics import YOLO + +model_dir = Path(os.environ["YOLO_MODEL_DIR"]) +datasets_dir = Path(os.environ["YOLO_DATASETS_DIR"]) + +# Tell ultralytics where to store datasets (needed for INT8 calibration) +os.environ["YOLO_DATASETS_DIR"] = str(datasets_dir) + +yolo_pt = model_dir / "yolo11n.pt" +int8_dir = model_dir / "yolo11n_int8_openvino_model" + +orig = os.getcwd() +os.chdir(str(model_dir)) +YOLO(str(yolo_pt)).export(format="openvino", int8=True, data="coco128.yaml") +# Older ultralytics exports INT8 to "yolo11n_openvino_model/" in CWD; +# rename it so it doesn't overwrite the FP32 export. +default_out = Path("yolo11n_openvino_model") +if default_out.exists() and not int8_dir.exists(): + default_out.rename(int8_dir.name) +os.chdir(orig) +print(f" INT8 quantization written to: {int8_dir}") +PYEOF + + _int8_ok=0 + for _isa in "" "AVX2"; do + if [ -n "${_isa}" ]; then + echo " ⚙ Retrying INT8 quantization with ONEDNN_MAX_CPU_ISA=${_isa} ..." + fi + set +e + env ${_isa:+ONEDNN_MAX_CPU_ISA="${_isa}"} \ + YOLO_MODEL_DIR="${YOLO_MODEL_DIR}" \ + YOLO_DATASETS_DIR="${YOLO_DATASETS_DIR}" \ + python3 "${_int8_script}" + _rc=$? + set -e + if [ "${_rc}" -eq 0 ] && [ -d "${YOLO_INT8_DIR}" ]; then + _int8_ok=1 + echo " ✓ INT8 quantization: ${YOLO_INT8_DIR}" + break + fi + if [ "${_rc}" -gt 128 ]; then + echo " ✗ INT8 quantization crashed with signal $(( _rc - 128 )) (exit ${_rc})." + else + echo " ✗ INT8 quantization failed (exit ${_rc})." + fi + # Remove any half-written IR so it is never loaded at runtime + rm -rf "${YOLO_INT8_DIR}" + done + rm -f "${_int8_script}" + + if [ "${_int8_ok}" -ne 1 ]; then + echo "" + echo " ⚠ WARNING: INT8 quantization could not be completed on this CPU." + echo " This is a known OpenVINO CPU-plugin crash during NNCF" + echo " calibration on CPUs the plugin does not yet recognise." + echo " Setup continues: the frame-selector automatically falls back" + echo " to the FP32 OpenVINO model, so take-away remains functional." + fi + fi + + echo "[3/3] Verifying YOLO artifacts..." _ok=1 if [ -f "${YOLO_PT}" ]; then echo " ✓ yolo11n.pt" @@ -736,12 +799,14 @@ PYEOF if [ -d "${YOLO_INT8_DIR}" ]; then echo " ✓ yolo11n_int8_openvino_model/" else - echo " ✗ yolo11n_int8_openvino_model/ missing" - _ok=0 + # Not fatal: CPU inference falls back to the FP32 IR. + echo " ⚠ yolo11n_int8_openvino_model/ missing — using FP32 on CPU" fi - if [ "${_ok}" -eq 1 ]; then + if [ "${_ok}" -eq 1 ] && [ -d "${YOLO_INT8_DIR}" ]; then echo "✓ YOLO models ready" + elif [ "${_ok}" -eq 1 ]; then + echo "✓ YOLO models ready (FP32 only — INT8 unavailable on this CPU)" else echo "✗ Some YOLO artifacts are missing — check ${YOLO_MODEL_DIR}" fi diff --git a/take-away/frame-selector-service/app/frame_selector.py b/take-away/frame-selector-service/app/frame_selector.py index df2aa9cc..484434fa 100755 --- a/take-away/frame-selector-service/app/frame_selector.py +++ b/take-away/frame-selector-service/app/frame_selector.py @@ -729,8 +729,18 @@ def _patched_compile(self, model_or_path, device_name=None, config=None, **kwarg logger.warning(f"Could not patch OpenVINO compile_model ({_e}); device selection relies on AUTO") if _target_device == "CPU": - _yolo_model_path = openvino_int8_path - logger.info("Loading INT8 OpenVINO model (CPU)") + # INT8 is preferred on CPU, but quantization can be unavailable on CPUs the + # OpenVINO CPU plugin does not yet recognise (setup_models.sh degrades to + # FP32 in that case — ITEP-94280). Fall back rather than failing to start. + if openvino_int8_path.exists(): + _yolo_model_path = openvino_int8_path + logger.info("Loading INT8 OpenVINO model (CPU)") + else: + _yolo_model_path = openvino_fp32_path + logger.warning( + f"INT8 model not found at {openvino_int8_path}; " + "falling back to FP32 OpenVINO model on CPU" + ) else: _yolo_model_path = openvino_fp32_path logger.info(f"Loading FP32 OpenVINO model ({_target_device})") From b83f86ee483a3af019638fbe0cb068cdc78a9510 Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Wed, 29 Jul 2026 19:04:52 +0530 Subject: [PATCH 09/10] fix(dine-in): align container user with host UID and correct OVMS model name ITEP-93258: the container ran as a fixed uid 1000 and chowned the bind-mounted results/ to it. On hosts where the invoking user is not uid 1000 (e.g. EMT) the host user then could not chmod or remove those files, so a second `make up` failed. The entrypoint now remaps the dlstreamer user to HOST_UID/HOST_GID before dropping privileges, so results/ stays owned by the invoking user. The remap uses usermod/groupmod and still execs via `gosu dlstreamer` by name, which preserves supplementary groups - notably `video`, which GPU access depends on. Remapping to root is refused, and the whole block is skipped when HOST_UID/HOST_GID are unset, so existing deployments are unaffected. Also fixes an unrelated startup defect found while validating: the OVMS model name in .env.example lacked the precision suffix that setup_models.sh writes into ovms-service/models/config.json, so every inference request returned HTTP 404 and validation produced zero results. Verified: remap correct for unset/matching/mismatched/root HOST_UID; video group retained; files written to a bind mount are owned by the host uid; full dine-in stack healthy with a live GPU validation returning accuracy 1.0 and metrics_status gpu_utilization "ok". Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dine-in/.env.example | 5 +++- dine-in/Makefile | 8 ++++++ dine-in/docker-compose.yaml | 8 ++++++ dine-in/entrypoint.sh | 53 ++++++++++++++++++++++++++++++++----- 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/dine-in/.env.example b/dine-in/.env.example index dcb9a7ba..e8924ed5 100755 --- a/dine-in/.env.example +++ b/dine-in/.env.example @@ -17,7 +17,10 @@ LOG_LEVEL=INFO # Service Endpoints # ----------------------------------------------------------------------------- OVMS_ENDPOINT=http://ovms-vlm:8000 -OVMS_MODEL_NAME=openbmb/MiniCPM-V-4_5 +# Must match the model name registered in ovms-service/models/config.json, +# which setup_models.sh generates with the precision suffix (e.g. -int4). +# A mismatch makes every OVMS request return HTTP 404. +OVMS_MODEL_NAME=openbmb/MiniCPM-V-4_5-int4 VLM_PRECISION=int8 TARGET_DEVICE=GPU SEMANTIC_SERVICE_ENDPOINT=http://semantic-service:8080 diff --git a/dine-in/Makefile b/dine-in/Makefile index c2dfc943..2ba856ed 100755 --- a/dine-in/Makefile +++ b/dine-in/Makefile @@ -77,6 +77,14 @@ DINEIN_IMAGE_NAME ?= intel/order-accuracy-dine-in DINEIN_TAG ?= latest DINEIN_IMAGE ?= $(DINEIN_IMAGE_NAME):$(DINEIN_TAG) +# Host user identity, passed to the container so the app user is remapped to +# match. Keeps bind-mounted results/ owned by the invoking user on hosts where +# that user is not uid 1000 (e.g. EMT). See ITEP-93258. +HOST_UID ?= $(shell id -u) +HOST_GID ?= $(shell id -g) +export HOST_UID +export HOST_GID + # Registry configuration # REGISTRY=false : Build and use local images # REGISTRY=true : Use images from registry (default for up, pull for build) diff --git a/dine-in/docker-compose.yaml b/dine-in/docker-compose.yaml index 1fe4d4c5..38589a21 100644 --- a/dine-in/docker-compose.yaml +++ b/dine-in/docker-compose.yaml @@ -93,6 +93,10 @@ services: - ./results:/app/results - ./configs:/app/configs:ro environment: + # Align the container's app user with the host user so bind-mounted + # results/ stays owned by the invoking user (ITEP-93258). + - HOST_UID=${HOST_UID:-1000} + - HOST_GID=${HOST_GID:-1000} - SEMANTIC_SERVICE_ENDPOINT=${SEMANTIC_SERVICE_ENDPOINT:-http://semantic-service:8080} - OVMS_ENDPOINT=${OVMS_ENDPOINT:-http://ovms-vlm:8000} - OVMS_MODEL_NAME=${OVMS_MODEL_NAME:-Qwen/Qwen2.5-VL-7B-Instruct} @@ -137,6 +141,10 @@ services: - ./results:/app/results - ./configs:/app/configs:ro environment: + # Align the container's app user with the host user so bind-mounted + # results/ stays owned by the invoking user (ITEP-93258). + - HOST_UID=${HOST_UID:-1000} + - HOST_GID=${HOST_GID:-1000} - WORKERS=${WORKERS:-1} - ITERATIONS=${ITERATIONS:-0} - REQUEST_DELAY=${REQUEST_DELAY:-0} diff --git a/dine-in/entrypoint.sh b/dine-in/entrypoint.sh index 5a7f6681..d46b8323 100755 --- a/dine-in/entrypoint.sh +++ b/dine-in/entrypoint.sh @@ -1,11 +1,52 @@ #!/bin/bash set -e -# Fix ownership of mounted volumes that Docker may create as root -# This runs as root before dropping to dlstreamer -if [ -d "/app/results" ]; then - chown -R dlstreamer:dlstreamer /app/results +# Align the in-container application user with the host user that owns the +# bind-mounted volumes. Without this, files written to ./results are owned by +# the container's UID (1000); on hosts where the invoking user is not uid 1000 +# (e.g. EMT) the host user then cannot chmod/remove them, which breaks a +# second `make up`. See ITEP-93258. +APP_USER=dlstreamer +TARGET_UID="${HOST_UID:-}" +TARGET_GID="${HOST_GID:-}" + +if [ "$(id -u)" = "0" ] && [ -n "$TARGET_UID" ] && [ -n "$TARGET_GID" ]; then + CURRENT_UID="$(id -u "$APP_USER")" + CURRENT_GID="$(id -g "$APP_USER")" + + # Never remap to root; that would defeat the privilege drop below. + if [ "$TARGET_UID" = "0" ] || [ "$TARGET_GID" = "0" ]; then + echo "entrypoint: refusing to remap $APP_USER to root; keeping ${CURRENT_UID}:${CURRENT_GID}" >&2 + elif [ "$TARGET_UID" != "$CURRENT_UID" ] || [ "$TARGET_GID" != "$CURRENT_GID" ]; then + # -o permits a non-unique id, in case it collides with an existing entry. + if [ "$TARGET_GID" != "$CURRENT_GID" ]; then + groupmod -o -g "$TARGET_GID" "$APP_USER" + fi + if [ "$TARGET_UID" != "$CURRENT_UID" ]; then + usermod -o -u "$TARGET_UID" "$APP_USER" + fi + echo "entrypoint: remapped $APP_USER ${CURRENT_UID}:${CURRENT_GID} -> ${TARGET_UID}:${TARGET_GID}" + # Re-own files still carrying the old ids. Restricted to paths owned by + # the previous uid/gid so large bind mounts are not rewritten wholesale. + find /app -xdev \( -uid "$CURRENT_UID" -o -gid "$CURRENT_GID" \) \ + -exec chown -h "$TARGET_UID:$TARGET_GID" {} + 2>/dev/null || true + if [ -n "$HOME" ] && [ -d "$HOME" ]; then + chown -R "$TARGET_UID:$TARGET_GID" "$HOME" 2>/dev/null || true + fi + fi +fi + +# Fix ownership of mounted volumes that Docker may create as root. +# Runs as root before dropping privileges. +if [ "$(id -u)" = "0" ] && [ -d "/app/results" ]; then + chown -R "$APP_USER:$APP_USER" /app/results 2>/dev/null || true +fi + +# Drop privileges and exec the CMD as the application user. Using the user name +# (not a numeric id) preserves supplementary groups such as `video`, which GPU +# access depends on. +if [ "$(id -u)" = "0" ]; then + exec gosu "$APP_USER" "$@" fi -# Drop privileges and exec the CMD as dlstreamer -exec gosu dlstreamer "$@" +exec "$@" From a7de303df24eb19f86385a28734ad7375cb706b6 Mon Sep 17 00:00:00 2001 From: TanmayeeSharvani22 Date: Wed, 29 Jul 2026 19:43:02 +0530 Subject: [PATCH 10/10] fix(dine-in): export MiniCPM at int4 to match the model name VLM_PRECISION was int8 while OVMS_MODEL_NAME ended in "-int4". The suffix is only a directory label; the actual weight format comes from --weight-format, so setup_models.sh produced INT8 weights in a directory named "-int4". Re-running setup_models.sh overwrote a previously INT4 export with INT8, regressing dine-in benchmark mean latency from 2274 ms to 3791 ms and TPS from 12.52 to 7.74. Decode is memory-bandwidth-bound, so doubling weight bytes (3.94 GB -> 7.57 GB) costs ~1.6x. Setting int4 restores it: mean 2216 ms, TPS 12.89, accuracy 1.0 (8/8), image_max_size 448 unchanged throughout. The export now reports dtype=int4, bits=4, group_size=128 with u4 weights. Note: setup_models.sh skips export when the target directory already exists and does not compare precision, so changing VLM_PRECISION alone is a silent no-op - the stale export must be removed first. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dine-in/.env.example | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dine-in/.env.example b/dine-in/.env.example index e8924ed5..133353d8 100755 --- a/dine-in/.env.example +++ b/dine-in/.env.example @@ -21,7 +21,11 @@ OVMS_ENDPOINT=http://ovms-vlm:8000 # which setup_models.sh generates with the precision suffix (e.g. -int4). # A mismatch makes every OVMS request return HTTP 404. OVMS_MODEL_NAME=openbmb/MiniCPM-V-4_5-int4 -VLM_PRECISION=int8 +# Weight format passed to the model export (--weight-format). This must agree +# with the precision suffix in OVMS_MODEL_NAME above: the suffix is only a +# directory label, so setting int8 here while naming the model -int4 produces +# INT8 weights in an "-int4" directory (~1.6x slower decode). +VLM_PRECISION=int4 TARGET_DEVICE=GPU SEMANTIC_SERVICE_ENDPOINT=http://semantic-service:8080 API_TIMEOUT=60