ci.yml

name: ci
on:
  push:
    branches: [main]
  pull_request:

# Workflow-level: dedupe PR pushes only. The group is keyed on the SHA for a
# push, so two pushes to main never share a group and never cancel each other.
# That is deliberate: `documentation-checkpoint` and `commit-protocol-tag` below
# are DIFF-scoped over `github.event.before..github.sha`, so a cancelled main run
# leaves its own commit range permanently unvalidated (no later run re-covers it,
# because the next run's `before` is this run's `sha`).
#
# The expensive TREE-scoped jobs carry their own job-level group instead, which
# does collapse superseded main pushes. Net effect: a superseded push to main
# keeps only its 2 per-commit gates and drops the other 6 jobs.
#
# 2026-08-04: those two jobs DID carry `ci-doc-`/`ci-commit-` groups keyed on
# `github.ref` with `cancel-in-progress: true`, directly contradicting their own
# comments and this one. It was not theoretical: two consecutive pushes to main
# had their documentation-checkpoint and commit-protocol-tag cancelled by the
# next push, so commits cdec2d10 and 0435746d's ranges went unvalidated by the
# gates written to cover them. The groups are removed; these two jobs must never
# regain one. They are a checkout plus a Python script, so the cost of always
# running them is negligible against a silently skipped gate.
concurrency:
  group: ci-${{ github.event.pull_request.number || github.sha }}-${{ github.repository }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  agent-record:
    # Tree-scoped: validates the record as it stands at HEAD, so only the newest
    # push to a ref is meaningful. Cancellable.
    concurrency:
      group: ci-agent-record-${{ github.ref }}-${{ github.repository }}
      cancel-in-progress: true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Canonical roadmap tables and links are consistent
        run: |
          python3 scripts/check-agent-record.py
          python3 tests/scripts/test_agent_record.py
          python3 tests/scripts/test_doc_checkpoint.py
      - name: Accepted binary-release design and record anchors stay in sync
        run: |
          python3 scripts/check-release-binary-contract.py
          python3 tests/scripts/test_check_release_binary_contract.py
      - name: README stays a human-readable user-facing document
        run: |
          python3 scripts/check-readme-structure.py
          python3 tests/scripts/test_check_readme_structure.py
      - name: Benchmarks and features stay human-readable keyed tables
        run: |
          python3 scripts/check-public-doc-tables.py
          python3 tests/scripts/test_check_public_doc_tables.py
      - name: Architecture-support checklist matches the row states
        run: |
          python3 scripts/check-model-checklist.py
          python3 tests/scripts/test_check_model_checklist.py
      - name: Public supported-model list matches the C++ registry
        run: |
          python3 scripts/check-supported-models.py
          python3 tests/scripts/test_check_supported_models.py
      - name: Every production env var is documented or classified
        run: |
          python3 scripts/check-env-doc.py
          python3 tests/scripts/test_check_env_doc.py
      - name: Model add+RMSNorm glue routes through the fusion catalog
        run: |
          python3 scripts/check-fusion-consistency.py
          python3 tests/scripts/test_check_fusion_consistency.py
      - name: Model decode is born on the runner (device-resident logits)
        run: |
          python3 scripts/check-runner-routing-consistency.py
          python3 tests/scripts/test_check_runner_routing_consistency.py
      - name: Examples are thin clients of the public ABI, capabilities reach it
        run: |
          python3 scripts/check-surface-coverage.py
          python3 tests/scripts/test_check_surface_coverage.py
      - name: Critical regression tests remain executable and CTest-registered
        run: |
          python3 scripts/check-test-registration.py
          python3 tests/scripts/test_check_test_registration.py
      - name: cuBLASLt GEMM invocation stays dtype-faithful (template parity)
        run: |
          python3 scripts/check-gemv-invocation-consistency.py
          python3 tests/scripts/test_check_gemv_invocation_consistency.py
      - name: Protocol prose matches the checkers that enforce it
        env:
          EVENT_NAME: ${{ github.event_name }}
          PR_BASE: ${{ github.event.pull_request.base.sha }}
          PR_HEAD: ${{ github.event.pull_request.head.sha }}
          PUSH_BASE: ${{ github.event.before }}
          PUSH_HEAD: ${{ github.sha }}
        run: |
          set -eu
          python3 scripts/check-policy.py
          python3 scripts/check-prompt-contract.py
          python3 scripts/check-protocol-consistency.py
          python3 -m unittest \
            tests.scripts.test_policy_contract \
            tests.scripts.test_policy_waivers \
            tests.scripts.test_check_commit_trailers
          python3 tests/scripts/test_check_prompt_contract.py
          python3 tests/scripts/test_check_protocol_consistency.py
          python3 tests/scripts/test_agent_gates.py
          cutover="$(tr -d '\n' < .agents/policy-cutover)"
          if [ "$EVENT_NAME" = "pull_request" ]; then
            base="$PR_BASE"
            head="$PR_HEAD"
          else
            base="$PUSH_BASE"
            head="$PUSH_HEAD"
          fi
          python3 scripts/check-commit-trailers.py \
            --range "$base..$head" --cutover "$cutover"
      - name: The state-log tail is anchored and chronological
        run: |
          python3 scripts/check-state-order.py
          python3 tests/scripts/test_check_state_order.py
      - name: NOW.md stays a short one-Read resume surface
        run: |
          python3 scripts/check-now-current.py
          python3 tests/scripts/test_check_now_current.py
      - name: A gated row may never lose a gate command that can FAIL
        # An EXACT PIN over the SET of rows whose spec names a runnable command
        # -- not a count, and not shrink-only. Not a count, because a count
        # cannot tell a row that LOST its command from one that legitimately
        # left the gated population, and the fix for the second reads as
        # lowering the number for the first. Not shrink-only, because the unit
        # test asserts the set EQUALS RUNNABLE_BASELINE, so GROWTH reds this job
        # too: any movement, up or down, re-pins the baseline in the same
        # change, naming the rows and the reason.
        run: |
          python3 scripts/check-gate-commands.py --check
          python3 tests/scripts/test_check_gate_commands.py
      - name: Agent role machinery and role discipline
        env:
          EVENT_NAME: ${{ github.event_name }}
          PR_BASE: ${{ github.event.pull_request.base.sha }}
          PR_HEAD: ${{ github.event.pull_request.head.sha }}
          PUSH_BASE: ${{ github.event.before }}
          PUSH_HEAD: ${{ github.sha }}
        run: |
          set -eu
          pending_args=()
          if [ "$EVENT_NAME" = "pull_request" ]; then
            base="$PR_BASE"
            head="$PR_HEAD"
            pending_args=(--pending-pr-head "$PR_HEAD")
          else
            base="$PUSH_BASE"
            head="$PUSH_HEAD"
          fi
          python3 scripts/check-role-discipline.py \
            --base "$base" --head "$head" "${pending_args[@]}"
          python3 tests/scripts/test_agent_role.py
          python3 tests/scripts/test_agent_onboard.py
      - name: Claim view, helper queue and PR reviewability
        run: |
          python3 scripts/claim-view.py --check
          python3 scripts/ready-for-helper.py --check
          python3 scripts/upstream-inventory.py --check
          python3 tests/scripts/test_claim_view.py
          python3 tests/scripts/test_upstream_inventory.py
      - name: Live-state rows are reconciled against Git reality
        # The classifier reads its evidence from `origin/main`. This job's
        # checkout already uses `fetch-depth: 0`, which DOES leave
        # `refs/remotes/origin/*` behind, so the ref is present today -- the
        # fetch makes that independent of the checkout block above rather than
        # a standing assumption about it, and refreshes the ref besides.
        # Without a resolvable `origin/main` the audit aborts by design
        # (require_origin_main): "no origin/main" would otherwise read as
        # "every ACTIVE row is abandoned", absence of information wearing
        # absence of work's face.
        #
        # The refspec is EXPLICIT and forced, not a bare `git fetch origin
        # main`. A bare fetch only updates a remote-tracking ref when a
        # CONFIGURED `remote.origin.fetch` refspec matches, and checkout sets
        # that config from its own inputs (at shallower settings it narrows to
        # the triggering ref, on a PR `refs/pull/N/merge`); with no match it
        # writes FETCH_HEAD only. Naming the refspec creates the ref whatever
        # the checkout is configured to do.
        run: |
          git fetch -q origin +refs/heads/main:refs/remotes/origin/main
          python3 scripts/audit-live-rows.py --check
          python3 tests/scripts/test_audit_live_rows.py
  documentation-checkpoint:
    # Gate: every code/test/benchmark/spike/lifecycle iteration refreshes both
    # user-facing status surfaces in that same commit, including void attempts.
    #
    # DIFF-scoped: deliberately carries NO concurrency group. Its verdict covers
    # this push's own `before..sha` range and nothing re-covers that range later,
    # so cancelling it would silently exempt those commits from the gate.
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Every feature checkpoint updates STATUS, BENCHMARKS and NOW
        env:
          EVENT_NAME: ${{ github.event_name }}
          PR_BASE: ${{ github.event.pull_request.base.sha }}
          PR_HEAD: ${{ github.event.pull_request.head.sha }}
          PUSH_BASE: ${{ github.event.before }}
          PUSH_HEAD: ${{ github.sha }}
        run: |
          set -eu
          if [ "$EVENT_NAME" = "pull_request" ]; then
            base="$PR_BASE"
            head="$PR_HEAD"
          else
            base="$PUSH_BASE"
            head="$PUSH_HEAD"
          fi
          python3 scripts/check-doc-checkpoint.py --base "$base" --head "$head"
          python3 scripts/check-now-current.py --base "$base" --head "$head"
          pending_args=()
          if [ "$EVENT_NAME" = "pull_request" ]; then
            pending_args=(--pending-pr-head "$PR_HEAD")
          fi
          python3 scripts/check-role-discipline.py \
            --base "$base" --head "$head" "${pending_args[@]}"
  commit-protocol-tag:
    # Gate: every NEW commit must carry the FOLLOWING_AGENTS_PROTOCOL trailer,
    # asserting the contributor read AGENTS.md. See .agents/ai-coding-assistants.md.
    #
    # DIFF-scoped: deliberately carries NO concurrency group, same reasoning as
    # documentation-checkpoint above. It walks `before..sha` per push, so a
    # cancelled run means those commits are never checked for the trailer.
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Every new commit carries FOLLOWING_AGENTS_PROTOCOL
        run: |
          set -eu
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            base="${{ github.event.pull_request.base.sha }}"
            head="${{ github.event.pull_request.head.sha }}"
          else
            base="${{ github.event.before }}"
            head="${{ github.sha }}"
          fi
          # Enforce on the FIRST-PARENT mainline (the commits the submitter
          # directly lands on the target branch, incl. merge commits which are
          # skipped below). Merged-in feature-branch commits are owned by the
          # submitter via the merge and are not re-checked here.
          # New branch / unreachable base (all-zero SHA): only check the tip commit.
          if ! git cat-file -e "${base}^{commit}" 2>/dev/null; then
            commits="$head"
          else
            commits="$(git rev-list --first-parent "${base}..${head}")"
          fi
          fail=0
          for c in $commits; do
            # Skip merge commits (>1 parent) — they are not authored content.
            if [ "$(git rev-list --parents -n1 "$c" | wc -w)" -gt 2 ]; then continue; fi
            if ! git log -1 --format=%B "$c" | grep -q 'FOLLOWING_AGENTS_PROTOCOL'; then
              echo "::error::commit $c is missing the FOLLOWING_AGENTS_PROTOCOL trailer — read AGENTS.md"
              git log -1 --oneline "$c"
              fail=1
            fi
          done
          if [ "$fail" -ne 0 ]; then
            echo "One or more commits lack FOLLOWING_AGENTS_PROTOCOL. See AGENTS.md / .agents/ai-coding-assistants.md."
            exit 1
          fi
          echo "OK: all new commits carry FOLLOWING_AGENTS_PROTOCOL."
  pr-size:
    # PR-only: helper PRs stay cheap to review, which is what stops the serial
    # operator from becoming the bottleneck. Non-row branches are reported only.
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Helper PRs stay reviewable
        run: |
          python3 scripts/check-pr-size.py \
            --base "${{ github.event.pull_request.base.sha }}" \
            --head "${{ github.event.pull_request.head.sha }}" \
            --branch "${{ github.event.pull_request.head.ref }}" \
            --pr-number "${{ github.event.pull_request.number }}"
  cuda-arch-features:
    # Configure-tier assertions on the CUDA per-arch FEATURE TABLE. Needs neither
    # a CUDA toolkit nor a GPU: it drives cmake/CudaArchFeatures.cmake directly,
    # so the resolution that decides which architectures get fp4-mma /
    # cutlass-nvfp4 / cutlass-fp8 / marlin-nvfp4 / fa2 (whose historical failure
    # mode was a SILENT capability drop) is checked on every push.
    #
    # Tree-scoped (drives cmake at HEAD): cancellable.
    concurrency:
      group: ci-cuda-arch-features-${{ github.ref }}-${{ github.repository }}
      cancel-in-progress: true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: CUDA feature table resolves per architecture
        run: cmake -P cmake/CudaArchFeaturesTest.cmake
  vulkan-spirv-freshness:
    # The Vulkan backend commits its SPIR-V (src/vt/vulkan/vulkan_spirv.h) instead
    # of compiling shaders at build time, so the build needs no shader toolchain on
    # any machine -- strictly better than llama.cpp, which requires glslc on every
    # build box. The trade is an obligation to regenerate by hand, and NOTHING
    # ENFORCED IT: `gen-vulkan-spirv.py --check` existed but no job ran it, so a
    # .comp edit without a regenerate shipped silently. That is the failure mode
    # this job closes, before the shader surface grows (BACKEND-VULKAN, VK-A1).
    #
    # The glslang DOWNLOAD URL is the pin. An exact version-string assertion was
    # deliberately NOT used: the committed SPIR-V was measured byte-identical under
    # both the 16.4.0 that produced it and the 16.5.0 pinned here, so gating on the
    # string would be brittle without buying anything. The URL being fixed is what
    # makes the run deterministic.
    #
    # Tree-scoped (checks HEAD's artifact): cancellable.
    concurrency:
      group: ci-vulkan-spirv-freshness-${{ github.ref }}-${{ github.repository }}
      cancel-in-progress: true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install pinned glslang
        run: |
          set -euo pipefail
          curl -fsSL -o /tmp/glslang.tar.gz \
            https://github.com/KhronosGroup/glslang/releases/download/16.5.0/glslang-16.5.0-linux-x86_64-release.tar.gz
          mkdir -p /tmp/glslang
          tar xzf /tmp/glslang.tar.gz -C /tmp/glslang
          /tmp/glslang/bin/glslang --version | head -1
          echo "/tmp/glslang/bin" >> "$GITHUB_PATH"
      - name: Committed SPIR-V is not stale
        run: python3 scripts/gen-vulkan-spirv.py --check
  build-test-vulkan:
    # The Vulkan backend had NO CI leg at all: `VLLM_CPP_VULKAN=ON` appeared
    # nowhere in this workflow, so tests/vt/test_vulkan_backend.cpp ran on no
    # machine. It rotted exactly as you would expect. Accelerator-seam row S5
    # (af0b21ba) gave unified-memory devices a portable CPU reference tier, which
    # means a missed GetOp no longer throws; the Metal sibling test was updated for
    # that as Metal work continued, and the Vulkan one was not, so its "the
    # unimplemented ops throw" assertion sat RED and INVISIBLE from S5 until
    # VK-A1 built the backend by hand (2026-08-06). That is the gap this closes.
    #
    # Runs GPU-FREE on `mesa-vulkan-drivers`' llvmpipe software ICD, which is the
    # arrangement the fan-out spike already recorded as working. The suite is
    # scoped to the Vulkan-specific targets rather than all of ctest: the full
    # suite is already covered by build-test-cpu, and this lane exists to keep the
    # Vulkan-only TUs compiled and executed, not to duplicate it.
    #
    # Tree-scoped: cancellable.
    concurrency:
      group: ci-build-test-vulkan-${{ github.ref }}-${{ github.repository }}
      cancel-in-progress: true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install the software Vulkan ICD
        # Loader + llvmpipe only. No shader toolchain: the backend consumes the
        # COMMITTED SPIR-V, which is the whole point of the committed-artifact
        # route, and the vulkan-spirv-freshness job above is what keeps it honest.
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers
      - name: Configure
        run: cmake -S . -B build-vulkan -DVLLM_CPP_BUILD_TESTS=ON -DVLLM_CPP_VULKAN=ON
      - name: Build
        # Bounded parallelism for the same reason as build-test-cpu: an unbounded
        # parallel link OOM-kills the runner.
        run: cmake --build build-vulkan -j 2 --target test_vulkan_backend test_backend_cross_device
      - name: Vulkan backend gate
        run: ./build-vulkan/tests/test_vulkan_backend
      - name: Cross-device numerics vs the CPU oracle
        run: ./build-vulkan/tests/test_backend_cross_device
  device-leakage:
    # The DSR RATCHET (work row `S1` of .agents/specs/accelerator-seam-audit.md).
    # Counts device-specific references in `src/vllm/` + `include/vllm/` — the
    # layer that is supposed to be device-agnostic — and fails on ANY increase
    # over scripts/device-leakage-baseline.json. A reduction must lower the
    # baseline in the SAME commit, so the number can only ever move down.
    #
    # It exists because the audit re-measured the leakage and found it had DRIFTED
    # UPWARD with no bad commit: DeepSeek-V2, Qwen3-Coder and the attention-registry
    # work each added a device test in passing. Leakage grows silently under
    # well-executed work, which is a job for a ratchet, not a cleanup.
    #
    # Needs neither a CUDA toolkit nor a GPU — pure static analysis, like the
    # cuda-arch-features job above. The mutation suite proves the checker actually
    # catches a planted leak; an unpoliced checker is worse than none.
    #
    # Tree-scoped: the ratchet compares HEAD against the committed baseline, so
    # only the newest push matters. Cancellable.
    concurrency:
      group: ci-device-leakage-${{ github.ref }}-${{ github.repository }}
      cancel-in-progress: true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Shared-layer device leakage does not grow (DSR ratchet)
        run: |
          python3 scripts/check-device-leakage.py --report
          python3 tests/scripts/test_device_leakage.py
  build-test-cpu:
    # Tree-scoped and the most expensive lane in the workflow. Cancellable.
    concurrency:
      group: ci-build-test-cpu-${{ github.ref }}-${{ github.repository }}
      cancel-in-progress: true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Configure
        run: cmake -S . -B build -DVLLM_CPP_BUILD_TESTS=ON
      - name: Build
        # Bounded parallelism: a bare `-j` lets Make link ALL test executables at
        # once, which OOM-kills the runner (ld signal 9) during the parallel link.
        run: cmake --build build -j 2
      - name: Test
        run: ctest --test-dir build --output-on-failure
  sanitize-cpu:
    # The DYNAMIC detector lanes (VLLM_CPP_SANITIZE, see CMakeLists.txt). The
    # plain build-test-cpu job above proves the suite PASSES; it cannot see a
    # read one past a tensor row, a use-after-free of an engine-owned container,
    # a signed-overflow index computation, or an unsynchronized access between
    # the serving threads — every one of which stays green under -O2 until it
    # silently corrupts a token stream. These jobs run the SAME suite with
    # ASan+UBSan and with TSan.
    #
    # Two jobs, not one: the runtimes are mutually exclusive. Both are CPU-tier
    # (the lane refuses to configure with the CUDA backend on) and both are
    # `continue-on-error` for their FIRST landing so a pre-existing finding
    # cannot block unrelated work — the finding is triaged, then this flag is
    # removed and the lane becomes binding. Removing it is tracked as the
    # closing step of the hardening-adoption row.
    strategy:
      fail-fast: false
      matrix:
        lane: ["address,undefined", "thread"]
    continue-on-error: true
    # Tree-scoped: cancellable. `matrix.lane` MUST be part of the group key, or
    # the two mutually-exclusive sanitizer legs of the SAME run would land in one
    # group and cancel each other, leaving only whichever started second.
    concurrency:
      group: ci-sanitize-cpu-${{ matrix.lane }}-${{ github.ref }}-${{ github.repository }}
      cancel-in-progress: true
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Configure
        run: |
          cmake -S . -B build-sanitize \
            -DVLLM_CPP_BUILD_TESTS=ON \
            -DVLLM_CPP_CUDA=OFF \
            -DVLLM_CPP_SANITIZE='${{ matrix.lane }}'
      - name: Build
        run: cmake --build build-sanitize -j 2
      - name: Test
        # ctest runs the suite serially: the sanitizer runtimes multiply peak RSS,
        # and a parallel run OOM-kills the runner before it reports a finding.
        env:
          UBSAN_OPTIONS: print_stacktrace=1
          ASAN_OPTIONS: detect_leaks=1:strict_string_checks=1
          # The production DevicePool deliberately retains scratch blocks. Its
          # detector lane uses exact allocations and real frees so ASan can
          # distinguish that cache from a leak and can see use-after-free.
          VT_POOL_BYPASS: "1"
        run: ctest --test-dir build-sanitize --output-on-failure