recipes: add vLLM DSV4-Pro agentic disagg references

configs/agentx_aiperf.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+# Self-contained AIPerf driver for AgentX trace sweeps. The dataset is selected
+# through the recipe environment, and the virtual environment must already be
+# mounted.
+
+AIPERF_DIR="${AIPERF_DIR:-/workspace/srt-slurm-sa/aiperf}"
+AIPERF_VENV="${AIPERF_VENV:-$AIPERF_DIR/.venv}"
+AIPERF_PYTHON="${AIPERF_VENV}/bin/python"
+AIPERF_CLI="${AIPERF_VENV}/bin/aiperf"
+AIPERF_FAILED_REQUEST_THRESHOLD="${AIPERF_FAILED_REQUEST_THRESHOLD:-0.10}"
+AIPERF_MAX_OSL="${AIPERF_MAX_OSL:-8192}"
+AIPERF_TRAJECTORY_START_MIN_RATIO="${AIPERF_TRAJECTORY_START_MIN_RATIO:-0.25}"
+AIPERF_TRAJECTORY_START_MAX_RATIO="${AIPERF_TRAJECTORY_START_MAX_RATIO:-0.75}"
+AIPERF_AGENTIC_CACHE_WARMUP_DURATION="${AIPERF_AGENTIC_CACHE_WARMUP_DURATION:-600}"
+AIPERF_WARMUP_GRACE_PERIOD="${AIPERF_WARMUP_GRACE_PERIOD:-1800}"
+AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING="${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-0}"
+AIPERF_USE_LEGACY_DYNAMO_SESSION_CONTROL="${AIPERF_USE_LEGACY_DYNAMO_SESSION_CONTROL:-0}"
+AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS="${AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS:-}"
+
+check_env_vars() {
+    local missing=0
+    for name in "$@"; do
+        if [ -z "${!name:-}" ]; then
+            echo "ERROR: required environment variable is missing: $name" >&2
+            missing=1
+        fi
+    done
+    if [ "$missing" -ne 0 ]; then
+        exit 1
+    fi
+}
+
+run_agentic_replay_and_write_outputs() {
+    local result_dir="$1"
+    mkdir -p "$result_dir"
+    env | sort > "$result_dir/env.txt"
+    printf '%s\n' "$REPLAY_CMD" > "$result_dir/aiperf_command.txt"
+    set +e
+    bash -lc "$REPLAY_CMD" 2>&1 | tee "$result_dir/aiperf.log"
+    local replay_rc="${PIPESTATUS[0]}"
+    set -e
+    return "$replay_rc"
+}
+
+check_env_vars MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION
+
+if [ ! -x "$AIPERF_PYTHON" ]; then
+    echo "ERROR: AIPerf Python is missing or not executable: $AIPERF_PYTHON" >&2
+    exit 1
+fi
+if [ ! -x "$AIPERF_CLI" ]; then
+    echo "ERROR: AIPerf CLI is missing or not executable: $AIPERF_CLI" >&2
+    exit 1
+fi
+
+"$AIPERF_PYTHON" - <<'PY'
+import aiperf
+import huggingface_hub
+
+print(f"Using aiperf {getattr(aiperf, '__version__', 'unknown')} from {aiperf.__file__}")
+print(f"Using huggingface_hub from {huggingface_hub.__file__}")
+PY
+
+AIPERF_PROFILE_HELP="$(COLUMNS=240 "$AIPERF_CLI" profile --help 2>&1)"
+
+RESULT_DIR="${RESULT_DIR:-/logs/agentic}"
+AGENTIC_OUTPUT_DIR="${AGENTIC_OUTPUT_DIR:-$RESULT_DIR}"
+PORT="${PORT:-8000}"
+AIPERF_BASE_URL="${AIPERF_BASE_URL:-http://localhost:$PORT}"
+MAX_CONTEXT_LENGTH="${MAX_CONTEXT_LENGTH:-1000000}"
+NUM_DATASET_ENTRIES="${NUM_DATASET_ENTRIES:-472}"
+PUBLIC_DATASET="${PUBLIC_DATASET:-semianalysis_cc_traces_weka_with_subagents_060826}"
+HF_WEKA_DATASET="${HF_WEKA_DATASET:-}"
+SLICE_DURATION="${SLICE_DURATION:-1.0}"
+VLLM_START_PROFILE_AFTER_SECONDS="${VLLM_START_PROFILE_AFTER_SECONDS:-900}"
+
+mkdir -p "$RESULT_DIR"
+
+export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES="${AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES:-0}"
+export AIPERF_DATASET_CONFIGURATION_TIMEOUT="${AIPERF_DATASET_CONFIGURATION_TIMEOUT:-1800}"
+export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT="${AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT:-1800}"
+
+REPLAY_CMD="$AIPERF_CLI profile"
+REPLAY_CMD+=" --scenario inferencex-agentx-mvp"
+REPLAY_CMD+=" --url $AIPERF_BASE_URL"
+REPLAY_CMD+=" --endpoint /v1/chat/completions"
+REPLAY_CMD+=" --endpoint-type chat"
+REPLAY_CMD+=" --streaming"
+REPLAY_CMD+=" --model $MODEL"
+REPLAY_CMD+=" --concurrency $CONC"
+REPLAY_CMD+=" --benchmark-duration $DURATION"
+REPLAY_CMD+=" --random-seed 42"
+REPLAY_CMD+=" --failed-request-threshold $AIPERF_FAILED_REQUEST_THRESHOLD"
+REPLAY_CMD+=" --trajectory-start-min-ratio $AIPERF_TRAJECTORY_START_MIN_RATIO"
+REPLAY_CMD+=" --trajectory-start-max-ratio $AIPERF_TRAJECTORY_START_MAX_RATIO"
+if [ -n "$AIPERF_AGENTIC_CACHE_WARMUP_DURATION" ] && [ "$AIPERF_AGENTIC_CACHE_WARMUP_DURATION" != "none" ]; then
+    REPLAY_CMD+=" --agentic-cache-warmup-duration $AIPERF_AGENTIC_CACHE_WARMUP_DURATION"
+fi
+if [ -n "$AIPERF_WARMUP_GRACE_PERIOD" ] && [ "$AIPERF_WARMUP_GRACE_PERIOD" != "none" ]; then
+    if grep -q -- "--warmup-grace-period" <<<"$AIPERF_PROFILE_HELP"; then
+        REPLAY_CMD+=" --warmup-grace-period $AIPERF_WARMUP_GRACE_PERIOD"
+    else
+        echo "NOTE: current aiperf does not support --warmup-grace-period; skipping AIPERF_WARMUP_GRACE_PERIOD=$AIPERF_WARMUP_GRACE_PERIOD." \
+            | tee "$RESULT_DIR/aiperf_warmup_grace_flag_note.txt"
+    fi
+fi
+REPLAY_CMD+=" --use-server-token-count"
+REPLAY_CMD+=" --no-gpu-telemetry"
+REPLAY_CMD+=" --tokenizer-trust-remote-code"
+if [ "$AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING" = "1" ]; then
+    if grep -q -- "--use-dynamo-conv-aware-routing" <<<"$AIPERF_PROFILE_HELP"; then
+        REPLAY_CMD+=" --use-dynamo-conv-aware-routing"
+    else
+        echo "ERROR: current aiperf does not support --use-dynamo-conv-aware-routing." >&2
+        exit 1
+    fi
+    if [ -n "$AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS" ]; then
+        if ! [[ "$AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
+            echo "ERROR: AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS must be a positive integer." >&2
+            exit 1
+        fi
+        if grep -q -- "--dynamo-session-timeout-seconds" <<<"$AIPERF_PROFILE_HELP"; then
+            REPLAY_CMD+=" --dynamo-session-timeout-seconds $AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS"
+        else
+            echo "ERROR: current aiperf does not support --dynamo-session-timeout-seconds." >&2
+            exit 1
+        fi
+    fi
+    if [ "$AIPERF_USE_LEGACY_DYNAMO_SESSION_CONTROL" = "1" ]; then
+        if grep -q -- "--use-legacy-dynamo-session-control" <<<"$AIPERF_PROFILE_HELP"; then
+            REPLAY_CMD+=" --use-legacy-dynamo-session-control"
+        else
+            echo "ERROR: current aiperf does not support --use-legacy-dynamo-session-control." >&2
+            exit 1
+        fi
+    fi
+elif [ "$AIPERF_USE_LEGACY_DYNAMO_SESSION_CONTROL" = "1" ]; then
+    echo "ERROR: legacy Dynamo session control requires AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=1." >&2
+    exit 1
+fi
+if grep -q -- "--vllm-start-profile-after-seconds" <<<"$AIPERF_PROFILE_HELP"; then
+    REPLAY_CMD+=" --vllm-start-profile-after-seconds $VLLM_START_PROFILE_AFTER_SECONDS"
+else
+    echo "NOTE: current aiperf does not support --vllm-start-profile-after-seconds; skipping it." \
+        | tee "$RESULT_DIR/aiperf_profile_flag_note.txt"
+fi
+if [ -n "$AIPERF_MAX_OSL" ] && [ "$AIPERF_MAX_OSL" != "none" ]; then
+    if grep -q -- "--trace-max-osl" <<<"$AIPERF_PROFILE_HELP"; then
+        REPLAY_CMD+=" --trace-max-osl $AIPERF_MAX_OSL"
+    elif grep -q -- "--synthesis-max-osl" <<<"$AIPERF_PROFILE_HELP"; then
+        REPLAY_CMD+=" --synthesis-max-osl $AIPERF_MAX_OSL"
+    else
+        echo "NOTE: current aiperf does not support max OSL capping; skipping AIPERF_MAX_OSL=$AIPERF_MAX_OSL." \
+            | tee "$RESULT_DIR/aiperf_max_osl_flag_note.txt"
+    fi
+fi
+if grep -q -- "--max-context-length" <<<"$AIPERF_PROFILE_HELP"; then
+    REPLAY_CMD+=" --max-context-length $MAX_CONTEXT_LENGTH"
+else
+    echo "NOTE: current aiperf does not support --max-context-length; skipping MAX_CONTEXT_LENGTH=$MAX_CONTEXT_LENGTH." \
+        | tee "$RESULT_DIR/aiperf_max_context_flag_note.txt"
+fi
+REPLAY_CMD+=" --num-dataset-entries $NUM_DATASET_ENTRIES"
+REPLAY_CMD+=" --slice-duration $SLICE_DURATION"
+REPLAY_CMD+=" --output-artifact-dir $RESULT_DIR/aiperf_artifacts"
+if [ "$DURATION" -lt 900 ] || [ "${AIPERF_UNSAFE_OVERRIDE:-false}" = "true" ]; then
+    REPLAY_CMD+=" --unsafe-override"
+fi
+if [ -n "$HF_WEKA_DATASET" ]; then
+    REPLAY_CMD+=" --hf-weka-dataset $HF_WEKA_DATASET"
+else
+    REPLAY_CMD+=" --public-dataset $PUBLIC_DATASET"
+fi
+
+run_agentic_replay_and_write_outputs "$RESULT_DIR"

configs/patches/vllm-mooncake-mtp-eagle-group-split-patch.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Keep same-spec eagle and non-eagle physical KV groups separate in Mooncake's
+# coordinator (see vllm_mooncake_mtp_eagle_group_split.py).
+# Transitional c188b96 workaround only. vllm-internal e5827fabc1 +
+# 89510b05dc are the official fixes; do not carry this patch into newer images.
+set -euo pipefail
+
+python3 /configs/patches/vllm_mooncake_mtp_eagle_group_split.py

configs/patches/vllm_mooncake_mtp_eagle_group_split.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Split mixed eagle/non-eagle Mooncake attention groups.
+
+DeepSeek V4 + MTP marks the physical KV group containing the MTP layer as an
+eagle group. In c188b96, Mooncake's coordinator merges physical groups solely
+by KVCacheSpec. That coalesces the eagle SWA group with three non-eagle SWA
+groups, even though their reachable-block proofs differ by one peek block.
+The merged pool requires every physical group to contain every candidate hash,
+so the non-eagle groups remove the eagle peek block before eagle verification
+can consume it. The resulting cross-attention-group intersection is always
+empty and the Mooncake master reports Get=0.
+
+Keep all upstream eagle semantics, but make eagle membership part of the
+coordinator's grouping key. This produces separate same-spec eagle and
+non-eagle attention groups whose proofs can converge correctly.
+
+Runs inside the worker container after vLLM is installed (post_install_script).
+It is deliberately anchored to the c188b96 source shape and fails closed when
+that shape changes.
+
+Transition only: vllm-internal PRs #275 (e5827fabc1) and #287 (89510b05dc)
+supersede this workaround. Do not upstream this group-split patch. Remove it
+when the runtime image contains both official fixes (89510b05dc or a
+descendant).
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+from pathlib import Path
+
+MARKER = "SRT patch(mtp-mooncake-eagle-group-split)"
+
+MODULE = "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator"
+
+OLD_GROUPING = """            for existing_spec, group_ids, existing_cls in attention_groups:
+                if existing_spec == spec:
+                    assert manager_cls is existing_cls
+                    group_ids.append(i)
+                    break"""
+
+NEW_GROUPING = """            # SRT patch(mtp-mooncake-eagle-group-split): groups with
+            # identical specs can still require different reachable-block
+            # proofs. Do not merge an eagle physical group with non-eagle
+            # groups, or the shared block-pool intersection drops its peek.
+            is_eagle_group = i in self.eagle_group_ids
+            for existing_spec, group_ids, existing_cls in attention_groups:
+                existing_is_eagle_group = group_ids[0] in self.eagle_group_ids
+                if (
+                    existing_spec == spec
+                    and existing_is_eagle_group == is_eagle_group
+                ):
+                    assert manager_cls is existing_cls
+                    group_ids.append(i)
+                    break"""
+
+
+def patch_text(text: str, source: str) -> str:
+    """Return patched coordinator source, failing on an unexpected shape."""
+    if MARKER in text:
+        return text
+
+    anchor_count = text.count(OLD_GROUPING)
+    if anchor_count != 1:
+        raise RuntimeError(
+            f"expected exactly one Mooncake grouping anchor in {source}, "
+            f"found {anchor_count}"
+        )
+
+    patched = text.replace(OLD_GROUPING, NEW_GROUPING, 1)
+    compile(patched, source, "exec")
+
+    required = (
+        MARKER,
+        "is_eagle_group = i in self.eagle_group_ids",
+        "existing_is_eagle_group = group_ids[0] in self.eagle_group_ids",
+        "existing_is_eagle_group == is_eagle_group",
+    )
+    missing = [needle for needle in required if needle not in patched]
+    if missing:
+        raise RuntimeError(f"patched source validation failed: missing {missing}")
+    return patched
+
+
+def resolve_installed_path() -> Path:
+    module = importlib.import_module(MODULE)
+    if module.__file__ is None:
+        raise RuntimeError(f"module {MODULE} has no __file__")
+    return Path(module.__file__)
+
+
+def run_self_test() -> None:
+    fixture = (
+        "def split(self, attention_groups, spec, manager_cls, i):\n"
+        "    if True:\n"
+        "        if True:\n"
+        f"{OLD_GROUPING}\n"
+    )
+    patched = patch_text(fixture, "<self-test>")
+    assert patched.count(MARKER) == 1
+    assert patched == patch_text(patched, "<self-test-idempotent>")
+    print("Mooncake eagle-group split patch self-test passed")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--path",
+        type=Path,
+        help="coordinator.py to validate/patch instead of importing installed vLLM",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="validate transformed syntax without writing the target",
+    )
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+
+    if args.self_test:
+        run_self_test()
+        return
+
+    path = args.path if args.path is not None else resolve_installed_path()
+    text = path.read_text()
+    patched = patch_text(text, str(path))
+
+    if text == patched:
+        print(f"Mooncake eagle-group split patch already applied -> {path}")
+        return
+    if args.dry_run:
+        print(f"Mooncake eagle-group split patch dry-run passed -> {path}")
+        return
+
+    path.write_text(patched)
+    if path.read_text() != patched:
+        raise RuntimeError(f"failed to verify patched contents in {path}")
+    print(f"Patched Mooncake coordinator eagle/non-eagle grouping -> {path}")
+
+
+if __name__ == "__main__":
+    main()