diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml index 729b3fb2f..8e9895a66 100644 --- a/.github/workflows/collectivex-sweep.yml +++ b/.github/workflows/collectivex-sweep.yml @@ -27,6 +27,11 @@ on: description: "Keep only shards in this comma-list of modes (normal, low-latency); low-latency is a decode-only EP8 addition; blank = all" type: string default: '' + fp8_consume: + description: "DIAGNOSTIC ONLY. 'dequant' puts the FP8 dequant back inside the chain, so staging is no longer hoisted; changes what is measured. Blank = native (the production model)." + type: choice + default: '' + options: ['', dequant] concurrency: group: cx-${{ github.ref }}-${{ inputs.backend }}-${{ inputs.only_sku }} cancel-in-progress: false @@ -132,6 +137,12 @@ jobs: COLLX_SHARD_SKU: ${{ matrix.sku }} COLLECTIVEX_CANONICAL_GHA: '1' COLLECTIVEX_SOURCE_SHA: ${{ github.sha }} + # Diagnostic hatch, "native" on every normal run. Reaches the ranks because the container + # srun exports the submitting environment; EPBackend reads it at class-body evaluation. + # Defaulted here rather than passed through empty: the harness fails closed on an + # unrecognised value, and a blank input would set the variable to "" -- which is not + # "unset", so os.environ.get's default never applies and every leg would die at import. + CX_FP8_CONSUME: ${{ inputs.fp8_consume || 'native' }} # Consolidated shards run one bounded build-group in one Slurm allocation; # every production pool accepts 300 minutes. Allocations release as soon as # the shard finishes. diff --git a/.github/workflows/test-collectivex.yml b/.github/workflows/test-collectivex.yml index 7e233ad36..e9b7fbd6c 100644 --- a/.github/workflows/test-collectivex.yml +++ b/.github/workflows/test-collectivex.yml @@ -38,11 +38,16 @@ jobs: run: | python -m pip install --upgrade pip pip install --index-url https://download.pytorch.org/whl/cpu torch + # ep_flashinfer's combine-model switch compares wheel versions with + # packaging.version; the benchmark image ships it, this runner does not. + pip install packaging # Every skip in this suite is torch-gated, so a missing torch turns the oracle # checks into silent passes. Fail here instead, where the cause is obvious. - - name: Verify torch is importable - run: python -c "import torch; print(torch.__version__)" + # packaging is not skip-gated at all — without it the FlashInfer combine-model + # tests ERROR, so name it here too rather than debugging it from a traceback. + - name: Verify torch and packaging are importable + run: python -c "import torch, packaging; print(torch.__version__)" - name: Run unit tests run: | diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md index 7715624d5..a6b8a60db 100644 --- a/experimental/CollectiveX/README.md +++ b/experimental/CollectiveX/README.md @@ -14,10 +14,12 @@ responsibility. The full measurement methodology is in [docs/methodology.md](doc The workload uses packed placement and one pinned `fixed-profile` resource configuration per backend/topology; there is no tuning sweep. Combine is always BF16; dispatch precision is a swept dimension — a BF16 control plus, on every backend whose FP8 dispatch is supported upstream -(DeepEP V2, MoRI, UCCL-EP), an FP8 dispatch, caller-prequantized in `normal` mode (in `low-latency` -the DeepEP and UCCL-EP kernels quantize internally from BF16; MoRI stays caller-prequantized). NCCL -EP is BF16-only this release, so it emits the control alone. Coverage is uniform routing only. Cases -run in one of two modes: +(DeepEP V2, MoRI, UCCL-EP, FlashInfer EP), an FP8 dispatch, caller-prequantized in `normal` mode (in +`low-latency` the DeepEP and UCCL-EP kernels quantize internally from BF16; MoRI stays +caller-prequantized; FlashInfer has no `low-latency` path). That caller-side quantize is charged +inside the measured dispatch, because a production forward pass pays it on the critical path. NCCL EP +is BF16-only this release, so it emits the control alone. Coverage is uniform routing only. Cases run +in one of two modes: - `normal` uses `layout-and-dispatch-v1`, rank-deduplicated token payloads, and activation-only, unweighted rank-sum combine. It runs the full decode and prefill ladders. @@ -27,17 +29,21 @@ run in one of two modes: which at the scoped EP8 run `cudaIpc` over NVLink rather than its CPU-proxy transport; on MoRI the `IntraNodeLL` kernel (single-call, pure-intranode, same compact layout and unweighted rank-sum combine as `IntraNode`). It is a decode-phase-only, per-SKU-capability-gated addition whose runnable set differs from `normal`'s, so - it is enabled from each SKU's `ll_backends` registry entry (currently DeepEP V2 EP8 on H100/H200/B200, - MoRI EP8 on MI300X/MI325X/MI355X, UCCL-EP EP8 on H100/H200/B200 only — UCCL's low-latency host + it is enabled from each SKU's `ll_backends` registry entry (currently DeepEP V2 at EP8 on + H100/H200 and at EP8 *and EP16* on B200 — the nscale bare-metal pool, whose gdrdrv-backed IBGDA + over native IB is what a low-latency scale-out needs and no virtualized pool has — and on + GB200/GB300, whose EP16 stays inside the MNNVL scale-up domain; + MoRI EP8 on MI300X/MI325X/MI355X; UCCL-EP EP8 on H100/H200/B200 only — UCCL's low-latency host assert `kNumMaxTopK + 1 <= num_warp_groups * num_warps_per_group` cannot hold on AMD, where `kNumMaxWarpGroups` is 16, since upstream raised `kNumMaxTopK` 9 -> 16 (uccl#1016, 2026-07-13) and our pin is six days later; the product is 16 for every CU count, so this is a dated regression rather than a hardware limit, and the AMD SKUs keep UCCL-EP normal mode without LL — and NCCL EP EP8 on all six NVIDIA SKUs, restored once the single-handle fix removed the [NVIDIA/nccl#2303](https://github.com/NVIDIA/nccl/issues/2303) signal aliasing that had wedged them. - B300, GB200 and GB300 carry the `candidate` NCCL EP as their *only* low-latency row, so those - three SKUs have no production decode coverage; DeepEP V2 emits no LL row at all on B300/GB200/GB300 — `_ll_runnable` adds - only runnable cells, so those walls are prose here, not classified matrix rows). + B300 carries the `candidate` NCCL EP as its *only* low-latency row, so it has no production + decode coverage; DeepEP V2 emits no LL row at all on B300 (the IBGDA address-handle wall in the + backend table below), and `_ll_runnable` adds only runnable cells, so that wall is prose here + rather than a classified matrix row). Scoped single-node EP8 runs over the intra-node NVLink/XGMI low-latency path (no `/dev/gdrdrv` needed — validated on H200 with it absent); NVSHMEM/IBGDA on the wire carries payload only on a multi-node scale-out (EP16) run; the legacy Buffer still @@ -47,9 +53,54 @@ Cases use a fixed timing profile from `configs/sweep.json`: 256 trials x 8 timed samples per component) with 32 synchronized full roundtrip warmups before each measured component at every trial/point. Component measurement order rotates each trial so every timed component occupies every position in the sequence; each iteration takes the cross-rank maximum before nearest-rank -p50/p90/p95/p99, and roundtrip p99 is the headline latency. A keyed BLAKE2b counter produces +p50/p90/p95/p99. A keyed BLAKE2b counter produces byte-identical routing and gate weights on every runtime. +Those components all measure **fresh entry** — the GPU is drained around every timed window — the +latency of an idle pipeline, not what a decode loop pays. So every row also carries the **chained +pair period**: 4 trials x (128 dispatch→combine pairs issued back-to-back with no host sync, first +16 dropped as pipeline fill) = 448 observations, reduced across ranks by median. Each trial runs +**two sibling chains** — a floors chain carrying only per-op events, then a period chain carrying +only the outer pair events — because the first version's single six-events-per-pair chain charged +its four inner `record()` calls into the period wherever the device outran the host, publishing a +~flat 10–30µs host constant as transport (+20–38% at T=1, fleet-wide). `components.pair_period` is +the headline latency for every row that carries one (released 2026-08-06 after the b200/h200/gb200 +hand references were confirmed against two-pass fleet artifacts), and `summarize.py` footnotes what +its starred columns hold either way. The floors chain publishes `chain_floor_us`, the +cross-rank minimum of each op's window; the period chain also yields `chain_health.pair_spread_us` +(cross-rank cadence proof), `interpair_gap_us` (the per-pair cost outside the published window — the +regression guard for instrumentation creeping back in, and the discriminator that keeps a +sync-dominated `period − Σfloors` gap from being misread as that defect returning; see the +methodology's `chain_floor_us` bullet for what each sign of that residual means) and +`settle_drift_us` (late-half minus +early-half period — the convergence proof `chain_drop` otherwise merely assumes). Chained per-op +*medians* are never published: inter-rank wait parks in whichever op window a rank blocks in, stable +per rank, arbitrary across ranks, and conserved only in the pair total. Nothing existing was renamed +or re-meant and the sweep `version` stays 1, so consumers key on the presence of +`components.pair_period`. + +The chained regime is checked twice over: each chain trial's own final combined output is compared +against a drained pair through the identical code path (`correctness.chain_last_output_passed`, +with the size of any difference in `correctness.chain_last_output_error`), and the full oracle runs +once per ladder point against the state the chain leaves behind +(`correctness.post_chain_state_passed`). The second always gates; the first gates only where the +chain stages per pair, and is `null` where staging is hoisted out of the chain — under the hoist +neither regime combines an input matching its own dispatch, so the two are not comparable. That +boundary was measured rather than assumed: identical h100 cases differ by 1000×–2966× the combine +tolerance with the hoist and by exactly zero without it. See the methodology's Correctness section. The null is a deliberate, bounded gap — an FP8-only, +free-running-only, stateless corruption would red no gate — and the `CX_FP8_CONSUME=dequant` hatch +is the standing probe for it. + +`roundtrip` means dispatch then combine — the transport — in every row. Expert-output staging sits +outside it and is reported separately as `stage`; under FP8 that component is harness scaffolding +standing in for the expert GEMM, which in production consumes FP8 operands natively rather than +materialising a BF16 copy, so `stage` must not be summed into a total or compared between backends. +Rows measured before this change carried the staging copy inside the chain for MoRI BF16 and +FlashInfer BF16, and the sweep `version` stays 1 across it, so +`implementation.stage_excluded_from_roundtrip` and whether a `stage` component is present are the +only way to tell the two generations apart. See [docs/methodology.md](docs/methodology.md) for the +full contract. + Correctness is checked against an implementation-independent oracle that reproduces the backend's two-level reduction — intra-scale-up-domain FP32, then a BF16 cast of each domain's partial for the scale-out send. The combine gate is a tight max elementwise relative error below `8 * 2^-8` @@ -76,19 +127,22 @@ scale-up domain. | Backend | Engine availability | Current scope | |---|---|---| -| DeepEP V2 | `production` — vLLM `--all2all-backend deepep_v2`, SGLang `--moe-a2a-backend deepep` | `normal` mode is PR #605 `ElasticBuffer` plus exact upstream #630 and #640 fixes: LSA for scale-up and GIN for x86 EP16 scale-out. FP8 dispatch via `use_fp8_dispatch` (blockwise e4m3fn) alongside BF16. `low-latency` mode is the legacy `deep_ep.Buffer` IBGDA decode kernels (per-expert padded layout, weighted combine, `use_fp8` e4m3fn), decode/EP8 only. B300 is an unsupported coverage row in `low-latency`: the legacy Buffer self-enables NVSHMEM IBGDA even for a single-node EP8 run, and on B300 address-handle creation fails (`ibgda.cpp:2234 Unable to create ah`), rc255 on all eight ranks. `NVSHMEM_DISABLE_IB=1` does not help — the Buffer re-enables IBGDA regardless, and the run fails identically with it set and unset (measured on b300-002 and b300-011) | +| DeepEP V2 | `production` — vLLM `--all2all-backend deepep_v2`, SGLang `--moe-a2a-backend deepep` | `normal` mode is PR #605 `ElasticBuffer` plus exact upstream #630 and #640 fixes: LSA for scale-up and GIN for x86 EP16 scale-out. FP8 dispatch via `use_fp8_dispatch` (blockwise e4m3fn) alongside BF16. `low-latency` mode is the legacy `deep_ep.Buffer` IBGDA decode kernels (per-expert padded layout, weighted combine, `use_fp8` e4m3fn), decode only — EP8 wherever enabled, plus EP16 on GB200/GB300 (inside the MNNVL domain) and on B200's nscale bare-metal pool (IBGDA over native IB rails with `/dev/gdrdrv`; the prior virtualized b200 pool could never run it). B300 is an unsupported coverage row in `low-latency`: the legacy Buffer self-enables NVSHMEM IBGDA even for a single-node EP8 run, and on B300 address-handle creation fails (`ibgda.cpp:2234 Unable to create ah`), rc255 on all eight ranks. `NVSHMEM_DISABLE_IB=1` does not help — the Buffer re-enables IBGDA regardless, and the run fails identically with it set and unset (measured on b300-002 and b300-011) | | MoRI | `production` — vLLM `--all2all-backend mori_*`, SGLang `--moe-a2a-backend mori` | `normal` mode uses the direct `IntraNode` kernel for scale-up EP8 on every CDNA SKU. EP16 is an unsupported coverage row on all three: the adapter pins `InterNodeV1` over 2x8 XGMI + RDMA, but its combine corrupts at the transport layer (ROCm/mori#475), so the registry ships `mori: [8]` and no EP16 case is dispatched. `low-latency` mode selects the `IntraNodeLL` decode kernel (single-call, pure-intranode, same compact layout and unweighted combine as `IntraNode`), decode/EP8 only. FP8 dispatch is caller-prequantized (per-SKU e4m3fnuz on gfx942, e4m3fn on gfx950); combine stays BF16 (`quant_type=none`) alongside BF16 dispatch | | UCCL-EP | `candidate` — no engine exposes a UCCL-EP selector | [UCCL](https://github.com/uccl-project/uccl) EP: a drop-in, API-identical DeepEP replacement whose CPU proxies issue GPUDirect RDMA over plain `libibverbs` (no NVSHMEM/IBGDA), with software message ordering, atomics, and flow control; scale-up is single-node `cudaIpc` over NVLink/XGMI (never MNNVL). `normal` mode is the legacy `Buffer` `dispatch`/`combine` (unweighted rank-sum); `low-latency` reuses the legacy `low_latency_dispatch`/`low_latency_combine` decode kernels (weighted combine), decode/EP8 only. FP8 dispatch is caller-prequantized in `normal` mode (blockwise e4m3fn, per-SKU e4m3fnuz on gfx942); in `low-latency` mode the caller sends BF16 and the decode kernel quantizes to e4m3 internally (`use_fp8`). Combine is BF16. Runs on NVIDIA and AMD (H100/H200/B200 + MI300X/MI325X/MI355X), EP8 scale-up. Cross-node EP16 is functional (the internode RDMA path connects and the light case passes correctness) but its CPU-proxy throughput overruns the standardized per-case wall-clock budget on heavy token counts, so EP16 is an unsupported coverage row for now | -| NCCL EP | `candidate` — NVIDIA's own library, but no engine exposes an NCCL-EP selector | [NCCL EP](https://github.com/NVIDIA/nccl/tree/master/contrib/nccl_ep): NVIDIA's native MoE dispatch/combine on the NCCL Device API — LSA (NVLink load/store) intra-node, GIN (GPU-Initiated Networking) inter-node — driven through the `nccl4py` bindings. `normal` mode selects the `HIGH_THROUGHPUT` algorithm (FLAT `[N, hidden]` receive, unweighted rank-sum combine); the `LOW_LATENCY` algorithm carries an EP8 `ll_backends` row on all six NVIDIA SKUs, restored once the single-handle fix removed the NVIDIA/nccl#2303 signal aliasing. BF16 only: `contrib/nccl_ep/RELEASE.md` says "No FP8 support", so no FP8 case is emitted. That note is worth re-testing rather than trusting — the C library at our pinned commit does read `inputs->scales` and switch on e4m3/e5m2, the two documented FP8 exclusions are expert-major layouts we do not use, and `NVIDIA/nccl` has not moved since 2026-06-11 while `NVIDIA/nccl-extensions` has replaced that row outright. NVIDIA-only and CUDA 13 only. EP8 scale-up on H100/H200/B200/B300 plus EP8 and EP16 on GB200/GB300, where EP16 stays inside the MNNVL scale-up domain. x86 EP16 scale-out is an unsupported coverage row: the cross-node GIN path faults inside `nccl_ep.cc` identically on RoCE and IB across four SKUs, a GDAKI limit rather than a fabric-selection one | -| FlashInfer EP | `production` — vLLM `--all2all-backend flashinfer_nvlink_one_sided` | [FlashInfer](https://github.com/flashinfer-ai/flashinfer) `MoeAlltoAll`: TensorRT-LLM's one-sided MNNVL all-to-all, where each rank writes tokens straight into its peers' workspace windows and combine reads them back — no send/recv pairing and no NVSHMEM. `normal` mode only (there is one kernel family; no separate decode path), BF16 only, and GB200/GB300 only, since the transport is MNNVL. EP8 and EP16, both inside the scale-up domain. Unlike every other backend here, its combine accumulates in the PAYLOAD dtype rather than FP32: wheels before 0.6.16 reduce the top-k contributions with a pairwise BF16 tree that rounds at every level, so the oracle models that reduction directly (`combine_reduction = "topk-slot-tree"`) instead of widening the tolerance. 0.6.16 moved the accumulator to FP32, and the adapter switches models on the installed version | +| NCCL EP | `candidate` — NVIDIA's own library, but no engine exposes an NCCL-EP selector | [NCCL EP](https://github.com/NVIDIA/nccl/tree/master/contrib/nccl_ep): NVIDIA's native MoE dispatch/combine on the NCCL Device API — LSA (NVLink load/store) intra-node, GIN (GPU-Initiated Networking) inter-node — driven through the `nccl4py` bindings. `normal` mode selects the `HIGH_THROUGHPUT` algorithm (FLAT `[N, hidden]` receive, unweighted rank-sum combine); the `LOW_LATENCY` algorithm carries an EP8 `ll_backends` row on all six NVIDIA SKUs, restored once the single-handle fix removed the NVIDIA/nccl#2303 signal aliasing. That LL decode ladder is clamped to T<=128, below its 256-slot receive: `nccl_ep`'s combine recv pipeline is a port of DeepEP's pre-#642 kernel and is missing the same shared-memory fence before `mbarrier_arrive`, which corrupted T=256 on GB300 in 1 of 5 executions — bimodal, healthy rows at 0.0039 relative error against 0.4704 on the failure. The fence is absent at NVIDIA/nccl master, so it is unfixed upstream. The clamp lowers exposure and is **not** a safety boundary: the fence is missing on every combine recv and T=256 is merely the rung with the most pipeline iterations, so lower rungs are less likely to hit the race rather than immune. Restore when a fixed wheel ships BF16 only: `contrib/nccl_ep/RELEASE.md` says "No FP8 support", so no FP8 case is emitted. That note is worth re-testing rather than trusting — the C library at our pinned commit does read `inputs->scales` and switch on e4m3/e5m2, the two documented FP8 exclusions are expert-major layouts we do not use, and `NVIDIA/nccl` has not moved since 2026-06-11 while `NVIDIA/nccl-extensions` has replaced that row outright. NVIDIA-only and CUDA 13 only. EP8 scale-up on H100/H200/B200/B300 plus EP8 and EP16 on GB200/GB300, where EP16 stays inside the MNNVL scale-up domain. x86 EP16 scale-out is an unsupported coverage row: the cross-node GIN path faults inside `nccl_ep.cc` identically on RoCE and IB across four SKUs, a GDAKI limit rather than a fabric-selection one | +| FlashInfer EP | `production` — vLLM `--all2all-backend flashinfer_nvlink_one_sided` | [FlashInfer](https://github.com/flashinfer-ai/flashinfer) `MoeAlltoAll`: TensorRT-LLM's one-sided MNNVL all-to-all, where each rank writes tokens straight into its peers' workspace windows and combine reads them back — no send/recv pairing and no NVSHMEM. `normal` mode only (there is one kernel family; no separate decode path), and GB200/GB300 only, since the transport is MNNVL. FP8 dispatch is caller-prequantized blockwise e4m3fn, carried as a fourth dispatch payload alongside its per-128-block FP32 scales, with the combine plane forced to BF16 — the C++ `toNvDataType` accepts only fp16/bf16/fp32 for combine, so an FP8 combine buffer would raise rather than corrupt. EP8 and EP16, both inside the scale-up domain. Unlike every other backend here, its combine accumulates in the PAYLOAD dtype rather than FP32: wheels before 0.6.16 reduce the top-k contributions with a pairwise BF16 tree that rounds at every level, so the oracle models that reduction directly (`combine_reduction = "topk-slot-tree"`) instead of widening the tolerance. 0.6.16 moved the accumulator to FP32, and the adapter switches models on the installed version | DeepEP V2 means the `ElasticBuffer` implementation introduced by [DeepEP PR #605](https://github.com/deepseek-ai/DeepEP/pull/605), not a newer legacy `Buffer` build. -The pinned source is the [PR #630](https://github.com/deepseek-ai/DeepEP/pull/630) head, whose parent -is the #605 merge tree, plus the exact one-line library matcher from upstream -[PR #640](https://github.com/deepseek-ai/DeepEP/pull/640). The first fixes pure scale-up -initialization when GIN is unavailable; the second prevents NCCL shared-memory mappings from being -misclassified as duplicate NCCL libraries. Scale-up cases request NCCL Device API LSA and fail closed +The pinned source is upstream `main`, which contains #605 along with +[PR #630](https://github.com/deepseek-ai/DeepEP/pull/630) (fixes pure scale-up initialization when +GIN is unavailable), [PR #640](https://github.com/deepseek-ai/DeepEP/pull/640) (stops NCCL +shared-memory mappings being misclassified as duplicate NCCL libraries), and +[PR #642](https://github.com/deepseek-ai/DeepEP/pull/642) (the low-latency combine fence that fixes +the Blackwell top-rung corruption of +[issue #700](https://github.com/deepseek-ai/DeepEP/issues/700)), which the previous pin — the #630 +head on the pre-merge #605 branch — predated. Scale-up cases request NCCL Device API LSA and fail closed unless the realized LSA team covers the full EP world. x86 EP16 scale-out cases instead require the hybrid path with GIN, two logical scale-out domains represented by two physical RDMA ranks, and eight scale-up ranks per domain; GB EP16 remains MNNVL scale-up and therefore uses LSA. Whether a given @@ -117,10 +171,12 @@ result artifacts carry only the fields listed in the methodology. Each SKU's Slurm and storage values come from its tracked baseline in the registry. An optional runner-local JSON document at `$XDG_CONFIG_HOME/inferencex/collectivex.json` or -`COLLECTIVEX_OPERATOR_CONFIG` overlays that baseline per field; unknown runners, fields, duplicate -keys, and non-JSON input fail closed, and configuration is never evaluated as shell. GHA passes no -operator secret, so a SKU runs entirely from its tracked baseline unless a runner-local document is -present. +`COLLECTIVEX_OPERATOR_CONFIG` overlays that baseline per field; a runner with no registry entry, an +unknown field, and non-JSON input all fail closed, and configuration is never evaluated as shell. +Duplicate JSON keys are NOT rejected — `json.load` keeps the last silently — and runner keys other +than the one being resolved are not validated, so a typo'd SKU name is ignored rather than +reported. GHA passes no operator secret, so a SKU runs entirely from its tracked baseline unless a +runner-local document is present. All public per-SKU platform data lives in the tracked `configs/platform_config.json` registry: architecture/product, container image and platform, fixed placement, launcher, runnable backend/EP diff --git a/experimental/CollectiveX/bench/ep_backend.py b/experimental/CollectiveX/bench/ep_backend.py index 2741e719a..ac5bb05f0 100644 --- a/experimental/CollectiveX/bench/ep_backend.py +++ b/experimental/CollectiveX/bench/ep_backend.py @@ -63,7 +63,7 @@ class EPBackend(abc.ABC): combine, recv_tokens, inspect_dispatch, combine_transformed); everything the driver and the oracles need beyond that is provided here. Combine is always BF16; an adapter that supports FP8 dispatch overrides - SUPPORTED_PRECISIONS and the semantic_payload/_encode_dispatch hooks. + SUPPORTED_PRECISIONS and the semantic_payload/_validate_quantizer hooks. """ name: str = "" @@ -77,12 +77,25 @@ class EPBackend(abc.ABC): # adapter that also sends an FP8-quantized dispatch payload widens this. SUPPORTED_PRECISIONS: tuple = ("bf16",) stage_device_work = False - combine_needs_redispatch = False - dispatch_needs_combine_cleanup = False - # Adapters that reduce activations and top-k weights independently must carry - # the complete local weighted expert sum in the activation tensor. + # Dispatch and combine form a single-use pair: every timed combine needs a fresh + # dispatch and every timed dispatch must be drained by a combine (double-buffered + # low-latency result tensors; MoRI/FlashInfer phase asserts). One flag, because a + # handle is either reusable or it is not -- no adapter has ever needed one + # direction without the other. + requires_fresh_pair = False + # Shape of the receive plane dispatch delivers -- "token-rank": one row per + # (source token, dest rank), rank-deduplicated; "token-expert": one row per + # (source token, expert) assignment (the low-latency padded layouts). Selects the + # correctness oracle and the artifact's `logical_copies.wire` label; independent + # of combine_weight_semantics (MoRI's IntraNodeLL is token-rank AND unweighted in + # low-latency mode). + receive_layout = "token-rank" + # WHERE the top-k gate weight enters the combine. "unweighted-rank-sum": the staged + # combine input carries the gate folded in -- adapters that reduce activations and + # top-k weights independently must carry the complete local weighted expert sum in + # the activation tensor -- and the kernel only sums. "weighted-kernel-sum": the + # kernel multiplies by the gate itself. Selects the expected-combine arithmetic. combine_weight_semantics = "unweighted-rank-sum" - roundtrip_only = False # Realized wire formats recorded in the artifact. Combine is always BF16; # dispatch_dtype is overridden per-run by an FP8 adapter (e.g. "fp8-e4m3fn"). dispatch_dtype = "bf16" @@ -92,9 +105,11 @@ class EPBackend(abc.ABC): # adapter sends 1 byte/value plus (for a blockwise codec) per-block FP32 scales. dispatch_value_bytes = 2 dispatch_scale_bytes_per_copy = 0 - # Handle attribute stage() populates with the tensor combine sends. Every adapter must - # point this at the tensor its combine() reads (NCCL EP names it differently). - combine_input_attr = "combine_input" + # Handle contract, not an attribute of this class: every adapter's stage() sets + # handle.combine_input to the tensor its combine() reads. The value need not be a torch + # tensor -- nccl-ep stores its own nccl.ep wrapper -- because the shared paths below + # only ever pass it through. + # Which production FP8 consumption path the chained roundtrip models. # # native (default) - the expert consumes the dispatched fp8 + per-128-block scales @@ -110,49 +125,105 @@ class EPBackend(abc.ABC): # # `dequant` is a VERIFICATION HATCH, not a second metric: never a sweep axis, never a # default. It is retained because it costs nothing (BF16 needs the same staged-is-None - # branch) and because it reproduces historical numbers exactly for regression checks -- - # measured 302.0us against 302.5us in run 30177021271 at T=1. - # - # A second measured mode is unnecessary because the mismatched-config cost is DERIVABLE - # from what every run already emits: - # - # dequant roundtrip ~= roundtrip + stage (+2.4% .. -0.1%, b200 LL fp8 ladder) + # branch) and because it still reproduces historical deepep-v2/uccl-ep numbers for regression + # checks -- 302.0us against 302.5us in run 30177021271 at T=1. It does not reproduce MoRI fp8 + # (that stage now casts only the rows dispatch filled) or any pre-hoist BF16 roundtrip (the + # hatch is fp8-only by design). # - # slightly high because chaining amortises launch overhead (median rt/(d+s+c) = 0.93 - # across the corpus). The reverse does NOT hold -- reconstructing native as - # `dequant - stage` errs by -11.6% at T=1, -5% at T=64, and only converges by T=256, - # i.e. it is worst exactly in the decode regime the headline reports. So measure native - # and derive dequant, never the other way round. + # A second measured mode is unnecessary: `dequant roundtrip ~= roundtrip + stage` holds to + # within a few percent, so the mismatched-config cost is derivable from what every run + # already emits. Measure native and derive dequant, NEVER the reverse -- reconstructing + # native as `dequant - stage` is worst exactly in the decode regime the headline reports. + # The derivation-accuracy ladder is in docs/methodology.md (search "fp8_consume"). # - # It matters because for deepep-v2 and uccl `stage` is precisely the fp8 conversion - # (both set stage_device_work = self._fp8), so charging it to the chained roundtrip - # compares fp8 and bf16 through structurally different pipelines. On run 30177021271 - # that inverted the fp8-vs-bf16 verdict in 39 of 51 comparisons. dispatch and combine - # were always measured stage-free; only the chained roundtrip mixed it in. + # It matters because for deepep-v2 and uccl `stage` IS the fp8 conversion (both set + # stage_device_work = self._fp8), so charging it to the chained roundtrip compares fp8 and + # bf16 through structurally different pipelines: on run 30177021271 that inverted the + # fp8-vs-bf16 verdict in 39 of 51 comparisons. fp8_consume = os.environ.get("CX_FP8_CONSUME", "native") + if fp8_consume not in ("native", "dequant"): + raise ValueError(f"CX_FP8_CONSUME must be 'native' or 'dequant', got {fp8_consume!r}") @property - def stages_fp8_natively(self) -> bool: - """Whether the chained roundtrip should skip the per-iteration `stage()`. - - Gated on precision, NOT on `stage_device_work` alone. The two are equivalent for - deepep-v2 and uccl, but MoRI sets `stage_device_work = self._fp8 or not - self._external_input`, so its scale-up kernels (IntraNode/IntraNodeLL) report True - for BF16 as well -- and their `stage()` really does run a device copy into the - registered combine-input buffer. That copy is not an fp8 dequant and nothing in this - change's evidence says it should leave the timed region, so BF16 keeps executing it - inline every iteration and its numbers are unmoved. - - (Whether MoRI's registered-buffer copy is a production cost or a harness artefact is - a real open question -- a native integration may have the expert GEMM write straight - into that buffer -- but it is a separate question from fp8 consumption, it applies to - both precisions equally, and it needs its own evidence.) + def stage_excluded_from_roundtrip(self) -> bool: + """Whether the chained roundtrip skips the per-iteration `stage()`. + + `roundtrip` must mean dispatch -> combine -- the transport, staging excluded -- in every + row or it cannot be compared across backends, so the answer is yes whenever `stage()` + does device work, regardless of precision. Gating on precision as well left staging + inside the roundtrip for MoRI BF16 scale-up and FlashInfer BF16 alone, against 800+ + transport-only rows. + + Gated on `stage_device_work` rather than applied blanket: where `stage()` is a bare + pointer assignment there is nothing to lift, and hoisting anyway would hand the + low-latency backends a view into their double-buffered receive, whose parity flips on + each timed re-dispatch -- combine would then read the stale-parity buffer. + + `CX_FP8_CONSUME=dequant` opts an fp8 run back into the inline stage, to model a stack + that really does dequantise between the two collectives (see `fp8_consume`). """ - return ( - self.precision == "fp8" - and self.stage_device_work - and self.fp8_consume == "native" - ) + if not self.stage_device_work: + return False + return not (self.precision == "fp8" and self.fp8_consume == "dequant") + + def fused_quantize(self, eager): + """The fp8 quantize the TIMED dispatch should call, keyed on mode. + + Production quantises bf16->fp8 once per forward pass, fused, just before dispatch, so + the harness compiles it once outside the timed window: charging the eager 9-launch + composite (19.2us H100, 53.6us MI300X, against ~1.5-4.9us compiled) would publish this + harness's kernel count rather than production's cost and flip fp8-vs-bf16 verdicts on + that basis. Low-latency keeps the eager form -- its dispatch quantises internally so that + cost is already in-window, and the oracle's payload gate must keep matching the eager + helper's bits. `dynamic=False` because a dynamic build measured 6.3x slower; the cache + limit is raised because ~20+ shapes exceed the default of 8 and overflow falls back to + eager SILENTLY. + """ + if self.mode == "low-latency": + return eager + import torch + + torch._dynamo.config.cache_size_limit = 64 + if hasattr(torch._dynamo.config, "fail_on_recompile_limit_hit"): + # Prefer a loud failure over a silent eager fallback if the limit is ever hit. + torch._dynamo.config.fail_on_recompile_limit_hit = True + return torch.compile(eager, dynamic=False) + + def assert_quantize_identity(self, eager, fused, x) -> None: + """Fail loudly, untimed, if the compiled quantize is not the eager one bit-for-bit. + + The oracle's payload gate is a `torch.equal` between the sender's [T, hidden] quantize + and the oracle's [receive_count, hidden] one, so identity has to hold per row across + batch sizes, not merely deterministically. Both properties were verified on-metal for + e4m3fn and e4m3fnuz; this check names a future toolchain regression here instead of + leaving an unexplained fleet-wide payload mismatch. + """ + if fused is eager: + return + import torch + + def bits(pair): + values, scales = pair + return values.view(torch.uint8), scales + + eager_values, eager_scales = bits(eager(x)) + fused_values, fused_scales = bits(fused(x)) + if not (torch.equal(eager_values, fused_values) + and torch.equal(eager_scales, fused_scales)): + raise RuntimeError( + "compiled fp8 quantize is not bitwise identical to the eager helper; the " + "oracle payload gate would fail fleet-wide" + ) + rows = min(int(x.shape[0]), 3) + if rows: + part_values, part_scales = bits(fused(x[:rows])) + whole_values, whole_scales = fused_values[:rows], fused_scales[:rows] + if not (torch.equal(part_values, whole_values) + and torch.equal(part_scales, whole_scales)): + raise RuntimeError( + "compiled fp8 quantize is not per-row invariant across batch sizes; the " + "oracle compares a different row count than the sender quantised" + ) def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) @@ -272,35 +343,33 @@ def semantic_payload(self, x): """ return x - def _encode_dispatch(self, x): - """Return (dispatch_payload, oracle_semantic) for the source activations x. - - Base identity: send x, no separate oracle payload (BF16). An FP8 adapter - returns the caller-prequantized dispatch payload and the dequantized BF16 the - oracle must expect after the backend's own dequant. + def _validate_quantizer(self, x) -> None: + """Per-shape hook, run untimed from make_problem. An FP8 adapter whose timed + dispatch calls a COMPILED quantizer overrides this to assert the compiled form + is bit-identical to the eager one (see assert_quantize_identity). The base has + no quantizer, and low-latency adapters keep the eager form, so neither checks. """ - return x, None def make_problem(self, T, idx, weights, x): """Assemble the per-shape problem namespace. - dispatch_x is the payload actually sent (x itself in BF16; the caller- - prequantized encoding under FP8). oracle_x, when set, is the dequantized BF16 - the combine oracle must expect, so the tight gate needs no tolerance change. + dispatch_x is always x: every adapter quantizes INSIDE dispatch, where production + pays it, so nothing is prequantized by the caller. oracle_x is semantic_payload(x) + -- identity in BF16, and the exact quant->dequant round-trip the wire performs under + FP8, so the combine gate stays tight without a tolerance change. Computing it here + also compiles this rung's quantizer shape outside the timed window. """ import torch - dispatch_x, oracle_semantic = self._encode_dispatch(x) - problem = types.SimpleNamespace( + self._validate_quantizer(x) + return types.SimpleNamespace( T=T, x=x, - dispatch_x=dispatch_x, + dispatch_x=x, + oracle_x=self.semantic_payload(x), topk_idx=idx.to(self._topk_idx_dtype()), topk_weights=weights.to(torch.float32), ) - if oracle_semantic is not None: - problem.oracle_x = oracle_semantic - return problem def _topk_idx_dtype(self): """Integer dtype the backend's kernels expect for top-k routing indices.""" @@ -310,47 +379,157 @@ def _topk_idx_dtype(self): # ---- Timing template methods ----------------------------------------------------- def timed_components(self): - """Components measured for this backend: roundtrip always; the rest unless - the backend exposes only a stateful paired round trip.""" - components = ["roundtrip"] - if not self.roundtrip_only: - components.extend(["dispatch", "combine"]) - if self.stage_device_work: - components.append("stage") + """Components measured for this backend: roundtrip, dispatch and combine + always; stage only when it launches device work.""" + components = ["roundtrip", "dispatch", "combine"] + if self.stage_device_work: + components.append("stage") return components - def warm(self, problem, count): + def warm(self, problem, count, stage_every=False): """Untimed synchronized full round trips (fabric/clock warm-up; cold-jump-safe). Caches the dynamic receive cardinality once so adapters never read a device scalar during a timed trial (the count is stable for a fixed routing trace). + + `stage_every` re-materialises the combine input on every iteration; the default hoists it + after the first, mirroring `benchmark_roundtrip`. Where staging is excluded from the chain + the timed region stages nothing, so warming it warms work the measurement never performs + -- ~247us per FP8 dequant against a 61us roundtrip, the leg's largest single cost. + `benchmark_stage` opts in, because there staging is the timed operation. """ import torch + staged = None for _ in range(count): handle = self.dispatch(problem) if not hasattr(problem, "recv_tokens"): problem.recv_tokens = self.recv_tokens(handle) - self.stage(problem, handle) + if staged is None: + self.stage(problem, handle) + if not stage_every and self.stage_excluded_from_roundtrip: + staged = handle.combine_input + else: + handle.combine_input = staged self.combine(problem, handle) torch.cuda.synchronize() def run_roundtrip(self, problem, staged=None): """One chained round trip; returns combined activations. - `staged` supplies a pre-materialised combine input so the conversion pass stays out of - the timed region (see `fp8_consume`). When it is None the stage runs inline, which is - the `dequant` fp8 model and the BF16 path -- free for the adapters whose - received buffer is already the combine input, real device work for the ones that - must place it (mori, flashinfer-ep; both declare `stage_device_work`). + `staged` supplies a pre-materialised combine input so staging stays out of the timed + region -- the default wherever `stage()` does device work (see + `stage_excluded_from_roundtrip`). It is None where `stage()` is a bare pointer + assignment, or under the `CX_FP8_CONSUME=dequant` hatch that wants it back in the chain. """ handle = self.dispatch(problem) if staged is None: self.stage(problem, handle) else: - setattr(handle, self.combine_input_attr, staged) + handle.combine_input = staged return self.combine(problem, handle) + def benchmark_chain(self, problem, warmup, iters, drop): + """Free-running dispatch->combine pairs, no host sync: a floors chain, then a period chain. + + This is what a serving stack pays: a decode loop never stops between layers, so entry + skew amortises across the chain instead of landing on one op the way `roundtrip`'s + drained windows charge it. The pairing is `run_roundtrip`'s, so paired-API backends stay + in contract; every backend is measured. + + Two chains, because per-op events inside a chained pair execute immediately on an idle + stream, landing the host's record() cost in the pair window: six events per pair + published a flat +10-30us host constant on every vendor (+20-38% at T=1, decaying with + T). So the floors chain carries op-window events only, the period chain one outer pair + with nothing between its two collectives, and `chain_health.interpair_gap_us` + (start-to-start median minus window median) guards that defect in-artifact. + + Only the pair period and the per-op minima are publishable: each rank's inter-rank wait + parks in whichever op window it blocks in while the period is conserved, so `run_sweep` + enforces pair -> cross-rank median, per-op -> cross-rank minimum, never a chained per-op + median or p99. + + `drop` discards each chain's head (pipeline fill, not period). The chain's own final + combined output is returned under `combined` -- cloned after the closing synchronize, so + the copy is untimed and detached from any double-buffered receive the next dispatch would + overwrite. `run_sweep` checks it against a drained pair through this same code path and + separately reruns the full expert oracle against the state the chain leaves behind; both + fold into the point's verdict. Interior pairs stay unvalidated by design -- each pair + overwrites its predecessor's output, and holding or reducing every output would put + device work inside the timed loops (see methodology, Correctness). + Free-running is safe fleet-wide: every backend double-buffers per dispatch or completes + each op on a reusable handle, and deepep-v2 NORMAL probed clean with 256 un-synced pairs + (T=128, EP8+EP16, both precisions, 2026-08-06, pin 01dc3aaa). Returns post-`drop` series + in microseconds: `pair` and `start_to_start` from the period chain (the latter one + element shorter), `dispatch` and `combine` from the floors chain. + """ + import torch + + self.warm(problem, warmup) + staged = None + if self.stage_excluded_from_roundtrip: + # The same hoist `benchmark_roundtrip` performs, so the chain is dispatch -> combine + # and nothing else. The `CX_FP8_CONSUME=dequant` hatch leaves `staged` None, putting + # the conversion inside the pair period and inside neither per-op window -- where + # work between the two collectives belongs. + handle = self.dispatch(problem) + self.stage(problem, handle) + staged = handle.combine_input + self.combine(problem, handle) # drain the pair backends require + torch.cuda.synchronize() + # Events are allocated BEFORE the loops: an allocation between two record() calls is host + # work inside a window meant to belong to the stream, a measurable fraction of the period + # at the bottom of the ladder. + def events(): + return [torch.cuda.Event(enable_timing=True) for _ in range(iters)] + + dispatch_start, dispatch_end = events(), events() + combine_start, combine_end = events(), events() + pair_start, pair_end = events(), events() + + # ---- Floors chain: op windows only, pair boundaries uninstrumented. ---- + for i in range(iters): + dispatch_start[i].record() + handle = self.dispatch(problem) + dispatch_end[i].record() + if staged is None: + self.stage(problem, handle) + else: + handle.combine_input = staged + combine_start[i].record() + self.combine(problem, handle) + combine_end[i].record() + torch.cuda.synchronize() + + # ---- Period chain: nothing between the pair's collectives but the pair itself. ---- + for i in range(iters): + pair_start[i].record() + handle = self.dispatch(problem) + if staged is None: + self.stage(problem, handle) + else: + handle.combine_input = staged + combined = self.combine(problem, handle) + pair_end[i].record() + torch.cuda.synchronize() + + def series(starts, ends): + return [ + start.elapsed_time(end) * 1000.0 # ms -> us + for start, end in zip(starts[drop:], ends[drop:]) + ] + + return { + "pair": series(pair_start, pair_end), + "start_to_start": series(pair_start[:-1], pair_start[1:]), + "dispatch": series(dispatch_start, dispatch_end), + "combine": series(combine_start, combine_end), + # The period chain's final combined output, produced IN the free-running regime. + # Cloned post-sync (untimed, stream-ordered ahead of any later dispatch) so the + # caller can compare it against a drained pair without racing the buffers. + "combined": combined.clone(), + } + def benchmark_component(self, component, problem, warmup, iters): """Measure one named component; every component gets the same warm-up first.""" if component == "roundtrip": @@ -368,15 +547,17 @@ def benchmark_roundtrip(self, problem, warmup, iters): self.warm(problem, warmup) staged = None - if self.stages_fp8_natively: - # Materialise the expert-output stand-in ONCE, untimed. A native fp8 stack has no - # separate conversion between dispatch and combine, so the chained measurement - # must not contain one. Routing is fixed for a ladder point, so the same staged - # tensor is valid for every iteration (for MoRI it IS the registered combine - # buffer, already filled). + if self.stage_excluded_from_roundtrip: + # Materialise the expert-output stand-in ONCE, untimed, so the chained measurement is + # dispatch -> combine and nothing else. Routing is fixed for a ladder point, so the + # same staged tensor is valid for every iteration -- MoRI's is the dispatch output at + # BF16 or a `[:rows]` BF16 cast under FP8, FlashInfer's the workspace combine region, + # which sits past the end of every dispatch receive plane. Read back through + # `handle.combine_input` rather than constructed, so an adapter's non-torch payload + # (nccl-ep) would round-trip unchanged if one ever reached here. handle = self.dispatch(problem) self.stage(problem, handle) - staged = getattr(handle, self.combine_input_attr) + staged = handle.combine_input self.combine(problem, handle) # drain the pair backends require torch.cuda.synchronize() return time_us(torch, lambda p=problem: self.run_roundtrip(p, staged), 0, iters) @@ -390,16 +571,16 @@ def finish_dispatch(hh, p=problem): self.stage(p, hh) self.combine(p, hh) - dispatch_needs_cleanup = self.dispatch_needs_combine_cleanup return time_us( torch, lambda p=problem: self.dispatch(p), 0, iters, - post=finish_dispatch if dispatch_needs_cleanup else None, + post=finish_dispatch if self.requires_fresh_pair else None, ) def benchmark_stage(self, problem, warmup, iters): import torch - self.warm(problem, warmup) + # Staging is the timed operation here, so it must be warmed on every iteration. + self.warm(problem, warmup, stage_every=True) def prep_stage(p=problem): return self.dispatch(p) @@ -413,7 +594,7 @@ def stage_op(hh, p=problem): return time_us( torch, stage_op, 0, iters, pre=prep_stage, post=(lambda hh, p=problem: self.combine(p, hh)) - if self.dispatch_needs_combine_cleanup else None, + if self.requires_fresh_pair else None, ) def benchmark_combine(self, problem, warmup, iters): @@ -426,7 +607,7 @@ def prep_combine(p=problem): self.stage(p, hh) return hh - if self.combine_needs_redispatch: + if self.requires_fresh_pair: return time_us( torch, lambda hh, p=problem: self.combine(p, hh), 0, iters, pre=prep_combine, ) diff --git a/experimental/CollectiveX/bench/ep_deepep_v2.py b/experimental/CollectiveX/bench/ep_deepep_v2.py index 672e0c658..a98f7de84 100644 --- a/experimental/CollectiveX/bench/ep_deepep_v2.py +++ b/experimental/CollectiveX/bench/ep_deepep_v2.py @@ -21,10 +21,20 @@ raise -# The source pin in runtime/common.sh is PR #605 head at the #630 fix; #640 is NOT in -# the fetched tree — runtime/stage.py applies it as a local rewrite before the build. This adapter no longer -# verifies the wheel's commit tag against the pin — it checks only that the loaded -# deep_ep exposes ElasticBuffer (the from-source PR #605 capability). +# The source pin in runtime/common.sh is upstream main, which carries #630 and #640. This +# adapter does not check the wheel's commit tag, only that the loaded deep_ep exposes +# ElasticBuffer. + +# Low-latency receive sizing, deliberately two numbers: _LL_BUFFER_CAP sizes the pre-allocated +# receive (and so the transport footprint and fp8 dequant volume), _LL_LADDER_CAP bounds which +# token counts are measured. Equal today, but kept separate so the ladder can be clamped around a +# kernel defect at one rung without moving the footprint (see `create_buffer`). +_LL_BUFFER_CAP = 256 +_LL_LADDER_CAP = 256 +assert _LL_LADDER_CAP <= _LL_BUFFER_CAP <= 511, ( + "the LL receive cap must fit NVSHMEM_QP_DEPTH=1024 ((cap + 1) * 2 <= 1024 => cap <= 511) " + "and the measured ladder must fit inside the buffer" +) def _fp8_cast_helpers(): @@ -47,8 +57,8 @@ def _ll_dequant_static(fp8, scales): ``[num_local_experts, cap*num_ranks, hidden]`` = (32, 2048, 7168) at EP8). The low-latency padded shape is constant on every dispatch, so a static (``dynamic=False``) compile fuses to one FP32 pass (~0.5 ms, 6.3x, bit-identical to the dynamic kernel on valid slots). The - dequant runs in every timed component's warmup and samples (~hundreds of thousands of - calls over the profile), so the dynamic kernel's per-call overhead overran the leg's + dequant runs in every timed `stage` sample and once per other component's warm-up, so the + call count is large enough that the dynamic kernel's per-call overhead overran the leg's wall-clock budget (all ranks SIGKILLed ~22 min in, no result); the static form brings FP8 low-latency inside the budget BF16 already meets. Padding slots decode to NaN in both forms (FP8 padding bytes) — harmless, because combine is handle-indexed and never reads @@ -128,7 +138,8 @@ class DeepEPV2Backend(EPBackend): SUPPORTED_MODES = ("normal", "low-latency") SUPPORTED_PRECISIONS = ("bf16", "fp8") stage_device_work = False - combine_needs_redispatch = False + requires_fresh_pair = False + receive_layout = "token-rank" combine_weight_semantics = "unweighted-rank-sum" def __init__(self, args, rank, world_size, local_rank, device): @@ -151,26 +162,31 @@ def __init__(self, args, rank, world_size, local_rank, device): # deep_ep.utils.math) so the timed stage() does no module lookup in the # measured region. self._to_fp8, self._cast_back = _fp8_cast_helpers() + # Normal/HT quantises inside the timed dispatch with the compiled form; low-latency + # keeps the eager helper, whose bits its in-kernel quantise matches. See fused_quantize. + self._quant = self.fused_quantize(self._to_fp8) if self.mode == "low-latency": # Legacy Buffer IBGDA decode path: a distinct kernel family whose combine # multiplies by the gate at the source (weighted), not an unweighted rank sum. self.kernel_generation = "legacy-buffer-ll" + self.receive_layout = "token-expert" self.combine_weight_semantics = "weighted-kernel-sum" # LL result tensors are double-buffered and single-use per dispatch (upstream: # "you cannot hold more than 2 low-latency kernels' result tensors at a single # moment"), so every timed combine needs a fresh dispatch and every timed # dispatch must be drained by its combine. - self.combine_needs_redispatch = True - self.dispatch_needs_combine_cleanup = True + self.requires_fresh_pair = True def buffer_cap(self, args): if self.mode == "low-latency": # LL pre-allocates a fixed [num_local_experts, cap * num_ranks, hidden] receive - # buffer, so cap is a hard per-rank dispatch-slot bound (the harness clamps the - # decode ladder to it and reports the dropped point). 256 sits well under the - # default NVSHMEM_QP_DEPTH ceiling ((cap + 1) * 2 <= 1024 => cap <= 511 with - # NVSHMEM_QP_DEPTH=1024) and is adjustable if the decode ladder needs more. - return 256 + # buffer, so the cap is a hard per-rank dispatch-slot bound; the harness clamps the + # ladder to it and records any dropped point in the artifact. This was clamped to 128 + # while DeepEP's low-latency combine stochastically corrupted the T=256 rung on + # Blackwell (issue #700, fixed upstream by #642); the pin now tracks main so the + # ladder runs full. If the top rung reds again, check the pin before assuming the + # defect returned -- clamping here is the containment lever either way. + return _LL_LADDER_CAP return None def create_buffer(self, spec): @@ -180,6 +196,16 @@ def create_buffer(self, spec): args, world_size = self.args, self.world_size self.max_tokens = spec.max_tokens_per_rank if self.mode == "low-latency": + # Size the LL buffer from the fixed cap, not from the clamped ladder: the receive + # footprint sets both the transport's memory traffic and the fp8 dequant volume + # (`_ll_recv_bf16` converts the whole padded receive), so following the ladder would + # shift every retained rung and break comparability with the published series. + if spec.max_tokens_per_rank > _LL_BUFFER_CAP: + raise RuntimeError( + f"low-latency ladder maximum {spec.max_tokens_per_rank} exceeds the LL " + f"buffer cap {_LL_BUFFER_CAP}" + ) + self.max_tokens = _LL_BUFFER_CAP self._create_ll_buffer(spec) return _require_runtime() @@ -242,14 +268,29 @@ def _create_ll_buffer(self, spec): raise RuntimeError( "invalid DeepEP LL runtime: deep_ep.Buffer.low_latency_dispatch is absent" ) - # Verified pinned signatures (commit fa8a9b16, deep_ep/buffers/legacy.py): + # Verified pinned signatures (commit 01dc3aaa, deep_ep/buffers/legacy.py): # Buffer.get_low_latency_rdma_size_hint(num_max_dispatch_tokens_per_rank, - # hidden, num_ranks, num_experts) -> int (staticmethod, line 175) + # hidden, num_ranks, num_experts) -> int (staticmethod, line 176) # Buffer(group, num_nvl_bytes=0, num_rdma_bytes=0, low_latency_mode=False, - # num_qps_per_rank=24, allow_nvlink_for_low_latency_mode=True, ...) (line 33) + # num_qps_per_rank=24, allow_nvlink_for_low_latency_mode=True, + # allow_mnnvl=False, explicitly_destroy=False, ...) (line 33) num_rdma_bytes = deep_ep.Buffer.get_low_latency_rdma_size_hint( self.max_tokens, args.hidden, world_size, args.experts ) + kwargs = {} + # On an MNNVL rack the scale-up fabric is NVLink across trays, but the legacy Buffer + # defaults `allow_mnnvl=False` and a False there self-sets NVSHMEM_DISABLE_MNNVL, so + # leaving it unset runs the low-latency kernels over IBGDA on exactly the systems whose + # fast path is MNNVL. Keyed on the reported topology, not the SKU name. + if str(getattr(args, "scale_up_transport", "")) == "mnnvl": + import inspect + if "allow_mnnvl" in inspect.signature(deep_ep.Buffer.__init__).parameters: + kwargs["allow_mnnvl"] = True + else: + raise RuntimeError( + "MNNVL scale-up needs deep_ep.Buffer(allow_mnnvl=...); this wheel lacks it, " + "so the low-latency path would silently run over IBGDA" + ) self.buffer = deep_ep.Buffer( self.group, num_rdma_bytes=num_rdma_bytes, @@ -257,6 +298,7 @@ def _create_ll_buffer(self, spec): num_qps_per_rank=num_qps_per_rank, allow_nvlink_for_low_latency_mode=True, explicitly_destroy=True, + **kwargs, ) def _ll_recv_bf16(self, recv_x): @@ -286,18 +328,14 @@ def _topk_idx_dtype(self): def semantic_payload(self, x): if not self._fp8: return x - return self._cast_back(*self._to_fp8(x)) + # Same callable the wire uses, so oracle and sender cannot disagree by construction. + return self._cast_back(*self._quant(x)) - def _encode_dispatch(self, x): - if not self._fp8: - return x, None - if self.mode == "low-latency": - # low_latency_dispatch takes BF16 x and casts to e4m3fn inside the kernel, so - # send x unquantized; expose the host round-trip as the oracle semantic so the - # combine expectation models the FP8 transport (same as semantic_payload). - return x, self._cast_back(*self._to_fp8(x)) - quantized = self._to_fp8(x) - return quantized, self._cast_back(*quantized) + def _validate_quantizer(self, x): + # Low-latency keeps the eager quantize (fused_quantize returns it unchanged), so + # _quant IS _to_fp8 there and there is nothing to cross-check. + if self._fp8 and self.mode != "low-latency": + self.assert_quantize_identity(self._to_fp8, self._quant, x) def _ll_dispatch(self, p): # Verified pinned signature (legacy.py:553): @@ -322,8 +360,11 @@ def _ll_dispatch(self, p): def dispatch(self, p): if self.mode == "low-latency": return self._ll_dispatch(p) + # Quantise here, not in make_problem: production runs one fused bf16->fp8 kernel per + # forward pass immediately before this collective, so the timed window must contain it. + dispatch_x = self._quant(p.dispatch_x) if self._fp8 else p.dispatch_x recv_x, recv_topk_idx, recv_topk_weights, handle, _ = self.buffer.dispatch( - p.dispatch_x, + dispatch_x, topk_idx=p.topk_idx, topk_weights=p.topk_weights, num_experts=self.args.experts, diff --git a/experimental/CollectiveX/bench/ep_flashinfer.py b/experimental/CollectiveX/bench/ep_flashinfer.py index 22a2eb1ac..6ca3bb782 100644 --- a/experimental/CollectiveX/bench/ep_flashinfer.py +++ b/experimental/CollectiveX/bench/ep_flashinfer.py @@ -58,17 +58,57 @@ _COMBINE_FP32_SINCE = (0, 6, 16) +def _wheel_has_fp32_combine(version: str) -> bool: + """Does this wheel accumulate combine in FP32, per `_COMBINE_FP32_SINCE`? + + Needs real version ordering, not a digit scrape: reading `0.6.16rc1` as 0.6.16 models FP32 + against a per-level-rounding kernel, which can exceed COMBINE_REL_TOL and red a correct run. + The opposite error costs a few ulps, so anything unparseable answers False. + """ + from packaging.version import InvalidVersion, Version + + try: + return Version(version) >= Version(".".join(str(n) for n in _COMBINE_FP32_SINCE)) + except InvalidVersion: + return False + + +# FP8 block size, matching the DeepSeek-V3 recipe every other FP8 backend here uses. +_FP8_BLOCK = 128 + + +def _blockwise_cast_to_fp8(x): + """Per-128-channel e4m3 quantize: (values [m, n], FP32 scales [m, n//128]). + + Local rather than shared on purpose: deepep-v2 and uccl-ep must match their own libraries' + in-kernel bits, while nothing here quantises in-kernel, so this copy only has to round-trip + self-consistently. Three contracts, not three copies of one. + """ + m, n = x.shape + blocks = x.view(m, -1, _FP8_BLOCK) + amax = blocks.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + values = (blocks * (448.0 / amax.unsqueeze(2))).to(torch.float8_e4m3fn).view(m, n) + return values, (amax / 448.0).view(m, -1) + + +def _blockwise_cast_back(values, scales): + """Inverse of _blockwise_cast_to_fp8, to BF16.""" + m, n = values.shape + blocks = values.view(m, -1, _FP8_BLOCK).float() + return (blocks * scales.view(m, -1, 1)).view(m, n).to(torch.bfloat16) + + class FlashInferEPBackend(EPBackend): name = "flashinfer-ep" maturity = "production" # vLLM --all2all-backend flashinfer_nvlink_one_sided # One kernel family; see the module docstring for why there is no low-latency mode. SUPPORTED_MODES = ("normal",) - # BF16 first. The combine side accepts fp8_e4m3fn/uint8 output dtypes and a - # use_low_precision accumulate, but dispatch FP8 needs the scale payload plumbed as a - # second input_payload and validated against the oracle's cast round-trip; not this pass. - SUPPORTED_PRECISIONS = ("bf16",) + # FP8 is dispatch-side only: scales ride as a fourth payload and combine stays BF16, so none + # of the 0.6.16+ combine-quant API is needed. vLLM accepts only nvfp4/mxfp8/bf16 on this + # transport, so an fp8 row measures the transport off-path; `dispatch_dtype` records that. + SUPPORTED_PRECISIONS = ("bf16", "fp8") kernel_generation = "flashinfer-mnnvl-one-sided" - # stage() now copies the received payload into the workspace combine region. + # stage() copies the received payload into the workspace combine region. stage_device_work = True # The kernel scatters expert outputs back to the supplying rank; it does not multiply by # the routing weights (those ride along as a caller payload, and vLLM applies them in the @@ -77,12 +117,19 @@ class FlashInferEPBackend(EPBackend): # Set per wheel in create_buffer; see _COMBINE_FP32_SINCE. combine_reduction = "topk-slot-tree" # Forced by the phase asserts described in the module docstring. - combine_needs_redispatch = True - dispatch_needs_combine_cleanup = True - combine_input_attr = "combine_input" + requires_fresh_pair = True def __init__(self, args, rank, world_size, local_rank, device): super().__init__(args, rank, world_size, local_rank, device) + self._fp8 = self.precision == "fp8" + if self._fp8: + # "-offpath" per SUPPORTED_PRECISIONS; bytes and block size match deepep-v2/uccl-ep. + self.dispatch_dtype = "fp8-e4m3fn-blockwise-offpath" + self.dispatch_value_bytes = 1 + self.dispatch_scale_bytes_per_copy = ( + (args.hidden + _FP8_BLOCK - 1) // _FP8_BLOCK + ) * 4 + self._quant = self.fused_quantize(_blockwise_cast_to_fp8) self._a2a = None self._max_tokens = None self.experts_per_rank = args.experts // world_size @@ -98,6 +145,16 @@ def _topk_idx_dtype(self): """ return torch.int32 + def semantic_payload(self, x): + if not self._fp8: + return x + # Same callable the wire uses, so sender and oracle cannot disagree by construction. + return _blockwise_cast_back(*self._quant(x)) + + def _validate_quantizer(self, x): + if self._fp8: + self.assert_quantize_identity(_blockwise_cast_to_fp8, self._quant, x) + def buffer_cap(self, args): # The workspace is sized from the ladder maximum rather than a fixed slot budget, so # there is no cap to clamp the ladder against. @@ -122,7 +179,11 @@ def create_buffer(self, spec): top_k = self.args.topk # Dispatch carries the activation plus the routing metadata the kernel needs per token: # int32 expert ids and fp32 gate weights, top_k of each. Combine carries BF16 hidden. - dispatch_bytes = hidden * 2 + top_k * 4 + top_k * 4 + dispatch_bytes = ( + hidden * self.dispatch_value_bytes + + self.dispatch_scale_bytes_per_copy + + top_k * 4 + top_k * 4 + ) combine_bytes = hidden * 2 workspace_size = moe_a2a_get_workspace_size_per_rank( ep_size=self.world_size, @@ -145,8 +206,8 @@ def create_buffer(self, spec): workspace_size_per_rank=workspace_size, mnnvl_config=MnnvlConfig(comm_backend=_communicator(_ep_group())), ) - wheel = tuple(int(n) for n in re.findall(r"\d+", flashinfer.__version__)[:3]) - if wheel >= _COMBINE_FP32_SINCE: + self.library_version = flashinfer.__version__ + if _wheel_has_fp32_combine(flashinfer.__version__): self.combine_reduction = "domain-fp32" # Every rank must finish mapping its workspace before any peer writes into it; # vLLM barriers here for the same reason. Scoped to the EP group, not the world. @@ -168,24 +229,58 @@ def dispatch(self, p): the tokens that selected one of its experts, so the kernel stamps the sentinel into the expert-id payload of every slot it did not fill. """ - recv_x, recv_idx, recv_w = self._a2a.dispatch( + # Quantise here, not in make_problem: production runs one fused bf16->fp8 kernel per + # forward pass immediately before this collective. The scales then ride as their own + # payload, which shifts the expert ids to index 2 -- four payloads, and the kernel's + # kMaxPayloads is exactly 4, matching vLLM's own [values, scales, ids, weights] order. + if self._fp8: + values, scales = self._quant(p.dispatch_x) + payloads = [values, scales, p.topk_idx, p.topk_weights] + expert_id_index = 2 + else: + payloads = [p.dispatch_x, p.topk_idx, p.topk_weights] + expert_id_index = 1 + received = self._a2a.dispatch( p.topk_idx, - [p.dispatch_x, p.topk_idx, p.topk_weights], + payloads, p.T, invalid_token_expert_id=_INVALID_EXPERT, - expert_id_payload_index=1, + expert_id_payload_index=expert_id_index, ) + # One received tensor per payload. Under FP8 `recv_x` stays the VALUES plane and the + # scales ride beside it, so every shape-sensitive consumer keeps working on a tensor. + if self._fp8: + recv_x, recv_scales, recv_idx, recv_w = received + else: + (recv_x, recv_idx, recv_w), recv_scales = received, None return types.SimpleNamespace( - recv_x=recv_x, recv_idx=recv_idx, recv_w=recv_w, + recv_x=recv_x, recv_scales=recv_scales, recv_idx=recv_idx, recv_w=recv_w, tokens=p.T, topk=p.topk_idx.shape[1], combine_input=None, ) def _combine_buffer(self, h): - """The workspace-resident combine payload region for this rung.""" + """The workspace-resident combine payload region for this rung. + + Always BF16: combine carries BF16 whatever the dispatch precision was, so this cannot + key off `recv_x.dtype` -- under FP8 that would size the region for 1-byte values. + """ return self._a2a.get_combine_payload_tensor_in_workspace( - h.tokens, h.recv_x.shape[-1], h.recv_x.dtype + h.tokens, h.recv_x.shape[-1], torch.bfloat16 ) + def _filled_slot_index(self, p, h): + """Row indices of the receive slots dispatch actually filled, resolved once per rung. + + Indexing with `_valid_rows`' boolean mask needs the match count on the host, which would + put a device read inside the timed stage. Routing is fixed per ladder point, so resolve to + an integer index on first use (always the untimed `warm()`) and cache it on the problem. + """ + index = getattr(p, "flashinfer_filled_slots", None) + if index is None: + index = self._valid_rows(h).nonzero(as_tuple=True)[0] + p.flashinfer_filled_slots = index + return index + def stage(self, p, h): """Materialise the combine payload in the workspace region the API designates. @@ -194,9 +289,23 @@ def stage(self, p, h): staging copy. Copying here rather than handing `combine` a caller-owned tensor keeps that copy out of the combine measurement, where production does not pay it; it is still executed and reported, as `stage`. + + Only the filled slots are copied, matching the kernel's own staging path + (`moeA2APrepareCombineKernel` returns early past `recv_counters[source]`); copying the + whole plane over-copied 1.5x at EP8 and 2.4x at EP16. Untouched slots are never read. """ buffer = self._combine_buffer(h) - buffer.copy_(h.recv_x) + filled = self._filled_slot_index(p, h) + hidden = h.recv_x.shape[-1] + flat_buffer = buffer.view(-1, buffer.shape[-1]) + source = h.recv_x.view(-1, hidden)[filled] + if self._fp8: + # Combine sends BF16, so the dequant lands here -- device work, hence `stage` is + # a reported component under either precision. + source = _blockwise_cast_back( + source, h.recv_scales.view(-1, h.recv_scales.shape[-1])[filled] + ) + flat_buffer[filled] = source h.combine_input = buffer def combine(self, p, h): @@ -229,6 +338,12 @@ def inspect_dispatch(self, p, h): keep = self._valid_rows(h) hidden = h.recv_x.shape[-1] payload = h.recv_x.reshape(-1, hidden)[keep] + if self._fp8: + # Dequantised with the same pair semantic_payload used, so the oracle compares like + # for like. + payload = _blockwise_cast_back( + payload, h.recv_scales.reshape(-1, h.recv_scales.shape[-1])[keep] + ) ids = h.recv_idx.reshape(-1, h.topk).to(torch.int64)[keep] weights = h.recv_w.reshape(-1, h.topk).to(torch.float32)[keep] local = (ids >= 0) & ((ids // self.experts_per_rank) == self.rank) diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py index ba34e135f..53462cc15 100644 --- a/experimental/CollectiveX/bench/ep_harness.py +++ b/experimental/CollectiveX/bench/ep_harness.py @@ -4,6 +4,7 @@ import argparse import datetime as _dt +from dataclasses import dataclass, field import json import math import os @@ -45,6 +46,12 @@ def case_id(sku: str, case: dict) -> str: # residual is the accumulation-order ambiguity the model cannot pin down: at most # topk (8) BF16 stores at one ulp (2^-8) each. Below the magnitude floor the gate # is effectively absolute (cancellation makes relative error meaningless there). +# +# The same bound covers the topk-slot-tree model (FlashInfer below 0.6.16, which rounds at +# every level of the reduction tree) without widening: a pairwise tree over topk=8 leaves is +# depth 3, so it compounds at most 3 roundings against the 8 this budget was sized for -- +# treewise summation is provably no worse than sequential. Both models are gated against this +# one constant, so a future model must be checked against it rather than assumed to fit. COMBINE_REL_TOL = 8 * 2.0 ** -8 COMBINE_MAG_FLOOR = 2e-2 @@ -62,6 +69,16 @@ def case_id(sku: str, case: dict) -> str: "low-latency": {"weighted-kernel-sum", "unweighted-rank-sum"}, } +# The (receive_layout, combine_weight_semantics) pairs the correctness oracles model. +# The two axes are independent declarations, but only these combinations have an +# expected-combine model; run_sweep fails closed on any other pairing. A backend +# declaring an unmodeled pair may be correctly implemented -- it needs an oracle +# written for it, not a silent verification against the wrong expectation. +ORACLE_MODELED_CONTRACTS = { + ("token-rank", "unweighted-rank-sum"), + ("token-expert", "weighted-kernel-sum"), +} + def logical_byte_provenance( logical_copies: int, hidden: int, @@ -134,6 +151,15 @@ def add_common_args(ap: argparse.ArgumentParser) -> None: help="timed iterations per trial") ap.add_argument("--trials", type=int, required=True, help="timed trials") + # Chain sampling on its own knobs: one call already yields chain_iters free-running pairs, so + # it converges in far fewer trials than the fresh-entry components. The matrix bakes these from + # configs/sweep.json `timing:`; the defaults match it, for cases scheduled before the fields. + ap.add_argument("--chain-iters", type=int, default=128, + help="free-running dispatch->combine pairs per chain trial") + ap.add_argument("--chain-trials", type=int, default=4, + help="chain trials per ladder point") + ap.add_argument("--chain-drop", type=int, default=16, + help="head pairs discarded per chain trial (pipeline fill, not period)") # provenance / output ap.add_argument("--runner", required=True) ap.add_argument("--topology-class", required=True) @@ -180,13 +206,25 @@ def _pcts(xs): "p95": percentile(xs, 95), "p99": percentile(xs, 99)} if xs else None) -def _component(percentiles, count, *, derived=False): +# Consumer contract, not labels: the frontend and durable store key the headline on +# `components.pair_period` carrying exactly CHAIN_PERIOD_ORIGIN, so a typo fails silently. +CHAIN_PERIOD_ORIGIN = "chained-median" +CHAIN_FLOOR_ORIGIN = "chained-cross-rank-min" + + +def _component(percentiles, count, *, derived=False, origin=None): + """One component block: availability, the reduction behind it, percentiles, sample count. + + `origin` names that reduction wherever it is not this suite's default per-iteration cross-rank + MAX. The chained families set it, because they share this block shape while being + differently-reduced statistics a consumer must not have to infer from the field name. + """ if percentiles is None: return {"availability": "unavailable", "origin": None, "percentiles_us": None, "sample_count": 0} return { "availability": "derived" if derived else "measured", - "origin": "derived-percentile-sum" if derived else "measured", + "origin": origin or ("derived-percentile-sum" if derived else "measured"), "percentiles_us": percentiles, "sample_count": 0 if derived else count, } @@ -237,13 +275,13 @@ def time_us(torch, fn, warmup: int, iters: int, pre=None, post=None) -> list[flo There is deliberately NO host sync between `pre()` and the start event. Stream ordering already keeps pre()'s work out of the s->e window: `s` is enqueued behind pre()'s kernels, - so the event timestamps when the stream REACHES it, not when the host recorded it. The sync - that used to sit here did not add that guarantee -- it drained the GPU, which put the host's - launch of `fn` inside the measured window and, because `fn` is a collective, let per-rank - launch jitter desynchronise ranks that pre() had just aligned. Each rank then blocked on the - slowest peer and run_sweep's cross-rank MAX reported that stagger as latency. Measured on - b200 uccl-ep low-latency combine: 113.4us -> 87.4us at T=1 with the ranks aligned, and no - change at T=32, which is what produced a *falling* latency curve as tokens grew. + so the event timestamps when the stream REACHES it, not when the host recorded it. A sync + here adds no guarantee and costs correctness: it drains the GPU, which puts the host's launch + of `fn` inside the measured window and, because `fn` is a collective, lets per-rank launch + jitter desynchronise ranks that pre() had just aligned. Each rank then blocks on the slowest + peer and run_sweep's cross-rank MAX reports that stagger as latency. Measured on b200 uccl-ep + low-latency combine: 113.4us with a sync against 87.4us without, at T=1, and no difference at + T=32 -- which is what produces a *falling* latency curve as tokens grow. The end-of-iteration sync stays: the warmup note below documents why iterations must not overlap (iter N+1's dispatch races iter N's combine on the persistent comm buffer), so only @@ -289,6 +327,37 @@ def _reduce_vec(torch, dist, device, vals, op): return [float(x) for x in t.tolist()] +def _reduce_vec_median_spread(torch, dist, device, vals): + """Per-element cross-rank (MEDIAN, MAX-MIN) for a chained series, from one all_gather. + + The pair period is a RATE, not a completion cost: every rank runs the same phase-locked + free-running loop, so MAX would publish whichever rank hiccuped as the pipeline's speed. The + median is the agreed cadence; a spread large next to it means one rank was paced -- distrust. + + One gather rather than three reductions, so the two cannot disagree about which iterations + they describe (and MEDIAN is not a `ReduceOp`); every rank gathers the same matrix in rank + order, so the artifact does not depend on which rank wrote it. + """ + local = torch.tensor(vals, device=device, dtype=torch.float64) + gathered = [torch.empty_like(local) for _ in range(dist.get_world_size())] + dist.all_gather(gathered, local) + stacked = torch.stack(gathered) + median = stacked.median(dim=0).values + spread = stacked.max(dim=0).values - stacked.min(dim=0).values + return [float(x) for x in median.tolist()], [float(x) for x in spread.tolist()] + + +def _gather_scalar(torch, dist, device, val): + """Every rank's copy of one per-rank scalar, in rank order, identical on every rank. + + The chain-health caller reduces it two ways (median and max-magnitude) from this one list. + """ + local = torch.tensor([float(val)], device=device, dtype=torch.float64) + gathered = [torch.empty_like(local) for _ in range(dist.get_world_size())] + dist.all_gather(gathered, local) + return [float(g.item()) for g in gathered] + + def _reduce_int(torch, dist, device, v: int, op) -> int: t = torch.tensor([int(v)], device=device, dtype=torch.int64) dist.all_reduce(t, op=op) @@ -532,6 +601,59 @@ def _oracle_report(**fields): return report +@dataclass +class PointSamples: + """Every sample series for one ladder point. + + Fresh-entry components carry one value per timed iteration, pooled across trials; + `spread` is the per-iteration cross-rank max-minus-min of the roundtrip; the `_min` + fields are the same iterations reduced by cross-rank MIN (the last rank into a + collective waited least, so its duration is the operation with entry skew excluded). + The chained family runs on its own much smaller trial count: `chain` and `chain_spread` + are per-iteration, while `gap` and `settle` are one scalar per trial. + """ + + dispatch: list = field(default_factory=list) + stage: list = field(default_factory=list) + combine: list = field(default_factory=list) + roundtrip: list = field(default_factory=list) + spread: list = field(default_factory=list) + dispatch_min: list = field(default_factory=list) + combine_min: list = field(default_factory=list) + roundtrip_min: list = field(default_factory=list) + chain: list = field(default_factory=list) + chain_spread: list = field(default_factory=list) + dispatch_floor: list = field(default_factory=list) + combine_floor: list = field(default_factory=list) + gap: list = field(default_factory=list) + settle: list = field(default_factory=list) + + +def _chain_output_matches(chained, drained): + """Whether the chain's final combined output matches a drained pair, same code path. + + A regime A/B, not an oracle: both tensors come from the backend's own dispatch->combine, + differing only in whether the pair ran free-running or drained, so a mismatch is corruption + that only manifests under back-to-back pairs (the stale-parity / aliased-signal class) -- + invisible to the drained oracles and to the fresh post-chain check alike. Judged with the + oracle's elementwise tolerance rather than bit equality because a combine kernel is not + required to be order-deterministic across invocations; a regime defect produces errors + orders of magnitude past COMBINE_REL_TOL, never inside it -- an assumption the returned + magnitude exists to CHECK rather than assert, since a verdict alone cannot distinguish a + corrupt result from one that merely landed just outside the tolerance. + + Returns (within_tolerance, worst_relative_error). + """ + if chained.shape != drained.shape: + return False, float("inf") + if not chained.numel(): + return True, 0.0 + error = (chained.float() - drained.float()).abs() + relative = error / drained.float().abs().clamp_min(COMBINE_MAG_FLOOR) + worst = float(relative.max().item()) + return worst < COMBINE_REL_TOL, worst + + def _run_expert_oracle( torch, routing, @@ -545,11 +667,11 @@ def _run_expert_oracle( seed: int, ): """Verify one real dispatch/transform/combine without entering a timed region.""" - # The low-latency decode kernels deliver a per-(source, expert) slot layout with a - # gate-weighted combine, which breaks this oracle's rank-deduplicated, unweighted - # assumptions. Route those cases to the dedicated per-slot oracle; keying on the - # declared combine semantics keeps the normal-mode path below untouched. - if getattr(backend, "combine_weight_semantics", None) == "weighted-kernel-sum": + # A per-(source, expert) slot receive breaks this oracle's rank-deduplicated + # assumptions. Route those layouts to the dedicated per-slot oracle; keying on the + # declared receive layout -- not the combine weighting, which is an independent + # declaration -- keeps the token-rank path below untouched. + if getattr(backend, "receive_layout", "token-rank") == "token-expert": return _run_ll_expert_oracle( torch, routing, backend, problem, global_idx, global_weights, rank, experts_per_rank, scale_up_domain, seed, @@ -857,6 +979,17 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> print(f"ERROR: iters/trials/warmup must be positive; got " f"{args.iters}:{args.trials}:{args.warmup}") return 2 + # Fail closed: a drop within one pair of the iteration count leaves nothing (or a single + # pair, whose start-to-start series is empty) and would publish `pair_period` degenerate and + # `chain_health` as "unavailable", indistinguishable from a backend that cannot be chained. + # Requiring two kept pairs here is what lets Pass 2b compute the health scalars + # unconditionally and Pass 3 assert the chained oracle ran. + if (min(args.chain_iters, args.chain_trials) <= 0 + or not 0 <= args.chain_drop <= args.chain_iters - 2): + if rank == 0: + print(f"ERROR: chain iters/trials must be positive and 0 <= drop <= iters - 2; got " + f"{args.chain_iters}:{args.chain_trials}:{args.chain_drop}") + return 2 import routing # torch-based; imported lazily so the module byte-compiles without torch ep_size = world_size @@ -881,6 +1014,21 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> f"backend declares {getattr(backend, 'combine_weight_semantics', None)!r}" ) return 2 + # Layout and weighting are declared independently; only two pairings have a + # correctness oracle (ORACLE_MODELED_CONTRACTS). Fail closed on the rest, so a + # backend whose declarations diverge errors with the missing model named instead + # of being verified against the wrong oracle and publishing the wrong wire basis. + declared_contract = ( + getattr(backend, "receive_layout", "token-rank"), + getattr(backend, "combine_weight_semantics", None), + ) + if declared_contract not in ORACLE_MODELED_CONTRACTS: + if rank == 0: + print( + "ERROR: no correctness oracle models receive_layout=" + f"{declared_contract[0]!r} with combine semantics {declared_contract[1]!r}" + ) + return 2 # A non-control precision must realize a non-BF16 dispatch wire format. Otherwise a # backend that lists the precision in SUPPORTED_PRECISIONS but never overrode its # encode hooks would run the case in BF16 and emit an artifact mislabeled with the @@ -953,46 +1101,57 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> "max_rel": oracle["max_elementwise_relative_error"] or 0.0, "local_ok": int(oracle["passed"]), "oracle_pre": oracle, + # Filled by Pass 2b after that point's last chain trial; the budget gate above + # guarantees at least one trial, so Pass 3 asserts this is no longer None. + "oracle_chain": None, + # ANDed across chain trials by Pass 2b: each trial's final chained output against + # a drained pair through the same code path. + "chain_output_local_ok": 1, + # Worst chained-vs-drained relative error seen at this point, kept even when the + # verdict passes: a magnitude creeping toward the tolerance is the early warning a + # bool cannot give, and the only way to tell a real corruption from a tight gate. + "chain_output_error": 0.0, "pre_input_unchanged": pre_input_unchanged, } + # The chained-output A/B is only defined when the chain stages per pair. Under the hoist + # (every FP8 adapter by default, since stage_device_work IS the fp8 flag) the staged + # stand-in is decoupled from each pair's dispatch, so chained and drained are not + # comparable -- see the call site for the measurement that established this. + chain_output_applicable = not backend.stage_excluded_from_roundtrip + # ---- Pass 2: every backend uses the same rotated point order. # Per-iteration cross-rank MAX samples are pooled across trials. ---- - disp_pool = {T: [] for T in ladder} # pooled per-iteration cross-rank MAX (dispatch) - stage_pool = {T: [] for T in ladder} # measured only when stage launches device work - comb_pool = {T: [] for T in ladder} # ... combine - rt_pool = {T: [] for T in ladder} # independently measured round trip - spread_pool = {T: [] for T in ladder} # cross-rank (max-min) of the round trip, per iter - # Cross-rank MIN per component. The LAST rank to enter a collective is the one that waited - # least -- it started when its peers were already there -- so its duration is the closest - # estimate of the operation's cost with entry skew excluded. MAX (reported as the latency) - # is that cost PLUS the skew, i.e. what the earliest-entering rank observed. - dmin_pool = {T: [] for T in ladder} - cmin_pool = {T: [] for T in ladder} - rtmin_pool = {T: [] for T in ladder} + # One object per ladder point, so a point's sample series travel together instead of as + # fourteen parallel dicts that must be kept in step by hand. Every list is per-iteration + # cross-rank reduced and pooled across trials; the reduction differs per field and is what + # the field name records (MAX for the published latencies, MIN for the skew-excluded floors, + # MEDIAN for the chained period -- see the reduction sites below). + samples = {T: PointSamples() for T in ladder} + for trial_index in range(args.trials): order = trial_order(list(ladder), trial_index) for T in order: problem = problems[T] - # timed_components() encodes the roundtrip-only vs full-component contract - # (and whether stage launches device work) once, in the base class. + # timed_components() encodes whether stage launches device work once, in + # the base class. component_order = trial_order(backend.timed_components(), trial_index) measured = {name: [] for name in ("dispatch", "stage", "combine", "roundtrip")} for component_name in component_order: # The base template gives every component the same synchronized - # full-roundtrip warm-up before its timed trial and encodes the two - # branch rules (dispatch cleanup, combine re-dispatch) internally. + # full-roundtrip warm-up before its timed trial and encodes the + # fresh-pair rule (dispatch drain, combine re-dispatch) internally. measured[component_name] = backend.benchmark_component( component_name, problem, args.warmup, args.iters ) # per-iteration cross-rank MAX (the distributed-op latency per iter), pooled. if measured["dispatch"]: - disp_pool[T] += _reduce_vec(torch, dist, device, measured["dispatch"], MAX) - comb_pool[T] += _reduce_vec(torch, dist, device, measured["combine"], MAX) + samples[T].dispatch += _reduce_vec(torch, dist, device, measured["dispatch"], MAX) + samples[T].combine += _reduce_vec(torch, dist, device, measured["combine"], MAX) if measured["stage"]: - stage_pool[T] += _reduce_vec(torch, dist, device, measured["stage"], MAX) + samples[T].stage += _reduce_vec(torch, dist, device, measured["stage"], MAX) rt_max = _reduce_vec(torch, dist, device, measured["roundtrip"], MAX) - rt_pool[T] += rt_max + samples[T].roundtrip += rt_max # Cross-rank SPREAD (max-min) of the same iterations. A collective cannot finish # before its slowest participant, so when ranks enter together every rank measures # nearly the same duration and the spread is small; a large spread means the ranks @@ -1000,11 +1159,88 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> # Emitted as a diagnostic so a skew-inflated point is visible in the artifact # instead of being mistaken for the operation getting slower. rt_min = _reduce_vec(torch, dist, device, measured["roundtrip"], MIN) - spread_pool[T] += [hi - lo for hi, lo in zip(rt_max, rt_min)] - rtmin_pool[T] += rt_min + samples[T].spread += [hi - lo for hi, lo in zip(rt_max, rt_min)] + samples[T].roundtrip_min += rt_min if measured["dispatch"]: - dmin_pool[T] += _reduce_vec(torch, dist, device, measured["dispatch"], MIN) - cmin_pool[T] += _reduce_vec(torch, dist, device, measured["combine"], MIN) + samples[T].dispatch_min += _reduce_vec(torch, dist, device, measured["dispatch"], MIN) + samples[T].combine_min += _reduce_vec(torch, dist, device, measured["combine"], MIN) + + # ---- Pass 2b: the chained family, on its own trial count. A separate loop because one call + # already yields chain_iters free-running pairs, so a handful of trials out-samples the + # fresh-entry components' 256 for a fraction of the wall clock. Ladder order still rotates + # per trial, as above. ---- + for trial_index in range(args.chain_trials): + final_chain_trial = trial_index == args.chain_trials - 1 + for T in trial_order(list(ladder), trial_index): + chained = backend.benchmark_chain( + problems[T], args.warmup, args.chain_iters, args.chain_drop + ) + # The chain's OWN final output against a drained pair through the identical + # dispatch->combine path, outside every timed region. This is the only chained + # output the run can inspect without putting device work inside the timed loops: + # each pair overwrites its predecessor's, so interior pairs are unvalidated by + # design (see methodology, Correctness). The drained pair is collective, issued in + # the same (trial, T) order on every rank, so the group stays aligned. + # + # Skipped entirely where staging is HOISTED, because there the comparison has no + # meaning. The hoist captures one warm-up dispatch's staged stand-in and reuses it + # for every pair, so neither the chain's final combine nor the drained reference + # consumes an input matching its OWN dispatch -- they are two differently + # mismatched pairs, and nothing requires them to agree. Measured, not assumed: + # h100/deepep-v2/EP8, identical in every other respect, native (hoisted) vs + # dequant (staged per pair) -- + # hoisted: chain_last_output_error 31..93 (1000x-2966x tolerance) + # per-pair: chain_last_output_error 0.0 (bit-identical, every rung) + # in BOTH normal and low-latency mode (runs 31180411148, 31185184372, 31185233991). + # Passing the chain's staged input to the drained pair was tried first and is NOT + # enough -- it makes the two share an input, but a shared input that matches + # neither dispatch. Only per-pair staging makes the regimes comparable, which is + # exactly the case this guard admits, so the drained pair stages inline here and + # `benchmark_chain`'s staged value has no consumer. Gating under the hoist reddened + # every FP8 leg fleet-wide for a harness artifact. + drained = backend.run_roundtrip(problems[T]) + torch.cuda.synchronize() + if chain_output_applicable: + output_ok, output_error = _chain_output_matches(chained["combined"], drained) + gate[T]["chain_output_local_ok"] &= int(output_ok) + gate[T]["chain_output_error"] = max( + gate[T]["chain_output_error"], output_error + ) + pair = chained["pair"] + pair_median, pair_spread = _reduce_vec_median_spread(torch, dist, device, pair) + samples[T].chain += pair_median + samples[T].chain_spread += pair_spread + # Per-op: cross-rank MIN only, from the FLOORS sibling chain (the period chain carries + # no per-op events). The chained windows park each rank's inter-rank wait, so only the + # minimum -- the last-entering rank's -- is the operation with the wait excluded. + samples[T].dispatch_floor += _reduce_vec(torch, dist, device, chained["dispatch"], MIN) + samples[T].combine_floor += _reduce_vec(torch, dist, device, chained["combine"], MIN) + # Chain-health scalars, one per trial. `interpair_gap_us` = start-to-start minus pair + # window: the per-pair cost outside the published window, the in-artifact guard against + # instrumentation self-charging. `settle_drift_us` = late-half minus early-half period. + # Median across ranks for the gap; signed max-magnitude for the drift. Unconditional: + # the budget gate guarantees two kept pairs, so both series are non-degenerate. + s2s_p50 = _pcts(chained["start_to_start"])["p50"] + gaps = _gather_scalar( + torch, dist, device, s2s_p50 - _pcts(pair)["p50"] + ) + samples[T].gap.append(_pcts(gaps)["p50"]) + half = len(pair) // 2 + drifts = _gather_scalar( + torch, dist, device, + _pcts(pair[half:])["p50"] - _pcts(pair[:half])["p50"], + ) + samples[T].settle.append(max(drifts, key=abs)) + if final_chain_trial: + # Gate the regime we publish: Passes 1 and 3 only check drained calls, so without + # this a backend that corrupts under free-running pairs would present as the + # fastest in the suite. Pass 3's machinery against the state this point's chain + # left behind, once per ladder point -- the failure mode is all-or-nothing. + idx_g, w_g = global_traces[T] + gate[T]["oracle_chain"] = _run_expert_oracle( + torch, routing, backend, problems[T], idx_g, w_g, rank, + experts_per_rank, scale_up_domain, args.seed, + ) # ---- Pass 3: prove timed inputs were immutable and repeat the full oracle. ---- for T in ladder: @@ -1021,12 +1257,35 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> scale_up_domain, args.seed, ) pre = gate[T]["oracle_pre"] + # The chained ORACLE is ANDed in like the other two, so a chained-regime failure reds the + # leg. The budget gate rejects chain_trials=0 up front, so a missing chained oracle is a + # harness bug, not a configuration. + chain_oracle = gate[T]["oracle_chain"] + assert chain_oracle is not None, "chained oracle missing despite a validated budget" + chain_ok = bool(chain_oracle["passed"]) + # The chained-OUTPUT check gates again, on a measured magnitude rather than a verdict. + # It was briefly demoted on the theory its tolerance was too tight for FP8; probe + # 31180411148 (h100, deepep-v2, EP8, low-latency) falsified that: + # bf16 chain_last_output_error = 0.0 at every rung -- bit-identical + # fp8 chain_last_output_error = 31..93 -- 1000x to 2966x COMBINE_REL_TOL + # A mis-set tolerance lands JUST outside; this is three orders of magnitude past, and + # the bf16 control proves the comparison itself is exact. Meanwhile every oracle passes + # (max_relative_error ~0.0039), which is precisely the signature this check exists for: + # a difference invisible to drained oracles. So FP8's chained output really does + # disagree with a drained pair, and a leg that cannot reproduce its own chained result + # should not publish a period from it. + chain_output_ok = bool(gate[T]["chain_output_local_ok"]) gate[T].update({ "input_unchanged": input_unchanged, - "local_ok": int(pre["passed"] and post["passed"] and input_unchanged), + "local_ok": int( + pre["passed"] and post["passed"] and chain_ok and input_unchanged + and (chain_output_ok or not chain_output_applicable) + ), + "chain_local_ok": int(chain_ok), "max_rel": max( pre["max_elementwise_relative_error"] or 0.0, post["max_elementwise_relative_error"] or 0.0, + chain_oracle["max_elementwise_relative_error"] or 0.0, ), "oracle_post": post, }) @@ -1037,7 +1296,7 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> gt = gts[T] g = gate[T] rstats = g["rstats"] - d, s, c, rt = disp_pool[T], stage_pool[T], comb_pool[T], rt_pool[T] + d, s, c, rt = samples[T].dispatch, samples[T].stage, samples[T].combine, samples[T].roundtrip dp, sp, cp, rtp = _pcts(d), _pcts(s), _pcts(c), _pcts(rt) # isolated_sum = SUM of the isolated dispatch+stage+combine percentiles. Stage contributes # zero when it is explicitly not applicable. This is NOT a measured chained operation @@ -1051,6 +1310,24 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> recv_max = _reduce_int(torch, dist, device, g["recv_local"], MAX) recv_min = _reduce_int(torch, dist, device, g["recv_local"], MIN) global_ok = _reduce_int(torch, dist, device, g["local_ok"], MIN) + # Agreed across ranks like `passed`, not rank 0's local view. + post_chain_state_passed = bool( + _reduce_int(torch, dist, device, g["chain_local_ok"], MIN) + ) + # null where the check does not apply (staging hoisted): the artifact says "not + # asked", never a bare False that a reader would mistake for a failed comparison. + # The reduce still runs on every rank so the collective stays aligned. + chain_last_output_passed = bool( + _reduce_int(torch, dist, device, g["chain_output_local_ok"], MIN) + ) + # Published whether or not the verdict passed. Without it the artifact records THAT the + # chained output differed but never BY HOW MUCH, which is the difference between a + # transport corruption and a tolerance set too tight for a backend's accumulator. + chain_output_error = _reduce_vec( + torch, dist, device, [g["chain_output_error"]], MAX + )[0] + if not chain_output_applicable: + chain_last_output_passed, chain_output_error = None, None max_rel = _reduce_vec(torch, dist, device, [g["max_rel"]], MAX)[0] point_ok = bool(global_ok) and recv_total > 0 throughput = { @@ -1066,33 +1343,92 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> backend.dispatch_value_bytes, backend.dispatch_scale_bytes_per_copy, ) combine_bytes = logical_byte_provenance(rstats["routed_copies"], args.hidden) + # Second byte basis, for backends whose wire carries one copy per (token, expert). Which + # applies is a property of the RECEIVE, not the mode -- MoRI's IntraNodeLL deduplicates + # where the other low-latency kernels do not -- so key it on the declared + # receive layout. `routed_copies` stays the canonical comparable basis. + assignment_copies = int(sum(rstats["expert_assignments_per_rank"])) + wire_basis = ( + "per-assignment" + if backend.receive_layout == "token-expert" + else "rank-deduplicated" + ) roundtrip_bytes = { field: dispatch_bytes[field] + combine_bytes[field] for field in dispatch_bytes } stage_bytes = dict.fromkeys(dispatch_bytes, 0) - spread = spread_pool[T] + spread = samples[T].spread + chain = samples[T].chain + chain_spread = samples[T].chain_spread + dfloor = samples[T].dispatch_floor + cfloor = samples[T].combine_floor + chain_gap = samples[T].gap + chain_settle = samples[T].settle + chainp = _pcts(chain) rows.append({ "components": { "combine": _component(cp, len(c)), "dispatch": _component(dp, len(d)), "isolated_sum": _component(isum, 0, derived=True), + # What a serving decode loop pays per MoE layer: the steady-state period of + # back-to-back dispatch->combine pairs, every backend, cross-rank median. Not + # `roundtrip` (drained around every pair, an idle-pipeline latency). Do not sum it. + "pair_period": _component(chainp, len(chain), origin=CHAIN_PERIOD_ORIGIN), "roundtrip": _component(rtp, len(rt)), "stage": _component(sp, len(s)), }, + # Per-op floors from the FLOORS sibling chain: cross-rank MINIMUM of each op's window, + # the last-entering rank's. Not the chained cost of dispatch/combine -- only the + # minimum excludes the parked wait. Tracks profiler kernel time to ~10%. + "chain_floor_us": { + "combine": _component(_pcts(cfloor), len(cfloor), origin=CHAIN_FLOOR_ORIGIN), + "dispatch": _component(_pcts(dfloor), len(dfloor), origin=CHAIN_FLOOR_ORIGIN), + }, + # Whether the chain was the steady state the period claims. `pair_spread_us` large + # next to `pair_period` means a paced rank; `interpair_gap_us` growth is the + # measurement loop contaminating the chain, not the fabric; `settle_drift_us` is the + # convergence proof `chain_drop` assumes but cannot show. Pass 2b has the reductions. + "chain_health": { + "interpair_gap_us": _component(_pcts(chain_gap), len(chain_gap)), + "pair_spread_us": _component(_pcts(chain_spread), len(chain_spread)), + "settle_drift_us": _component(_pcts(chain_settle), len(chain_settle)), + }, # Skew-excluded companion to `components`: same iterations reduced with cross-rank # MIN instead of MAX. Compare against `components` to see how much of a point is the # operation and how much is rank stagger; a curve that dips in MAX but not in MIN was # never the operation getting faster. "cross_rank_min_us": { - "combine": _component(_pcts(cmin_pool[T]), len(cmin_pool[T])), - "dispatch": _component(_pcts(dmin_pool[T]), len(dmin_pool[T])), - "roundtrip": _component(_pcts(rtmin_pool[T]), len(rtmin_pool[T])), + "combine": _component(_pcts(samples[T].combine_min), len(samples[T].combine_min)), + "dispatch": _component(_pcts(samples[T].dispatch_min), len(samples[T].dispatch_min)), + "roundtrip": _component(_pcts(samples[T].roundtrip_min), len(samples[T].roundtrip_min)), }, # Diagnostic, NOT a latency: per-iteration cross-rank (max-min) of the round trip. # Small => ranks entered together and the reported MAX is the operation's cost. # Large relative to the roundtrip => the point is skew-inflated; read it with care. "cross_rank_spread_us": _component(_pcts(spread), len(spread)), "correctness": { + # Whether the free-running chain's OWN final combined output (per trial) + # matched a drained pair through the identical code path, within the combine + # tolerance. Proves the last pair of each chain, not every interior pair -- + # validating those would put device work inside the timed loops. + # Folded into `passed` WHERE IT APPLIES; null where it does not, which is + # wherever staging is hoisted out of the chain (every FP8 adapter by default). + # There the staged stand-in is decoupled from each pair's dispatch, so chained + # and drained are not comparable and the question is not asked -- see the Pass + # 2b call site for the A/B that established that. Read it beside + # `chain_last_output_error`, never alone. + "chain_last_output_passed": chain_last_output_passed, + # How far apart they actually were (cross-rank MAX), published whether or not + # the verdict passed. A bool alone cannot separate a transport corruption from + # a tolerance too tight for a backend's accumulator, and reading it as the + # former without this number is a mistake this field exists to prevent. + "chain_last_output_error": chain_output_error, + # Whether the full oracle also passed against the state the free-running chain + # left behind (a FRESH dispatch+combine after the final trial). Renamed from + # `chain_regime_passed`, which overclaimed: older artifacts carry that name, + # and `null` there meant the chain never ran (a state the budget gate has since + # made impossible). Folded into `passed`. + "post_chain_state_passed": post_chain_state_passed, # Max elementwise relative error (COMBINE_MAG_FLOOR-clamped) # against the BF16-faithful expected combine. "max_relative_error": max_rel, @@ -1105,6 +1441,14 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> "roundtrip": roundtrip_bytes, "stage": stage_bytes, }, + # Copy counts behind the byte figures above, so a reader can rebase them: `routed` is + # the basis they use, `assignments` the per-(token, expert) count, `wire` which the + # kernels move. Kept out of `byte_provenance`, whose values are all per-component. + "logical_copies": { + "routed": int(rstats["routed_copies"]), + "assignments": assignment_copies, + "wire": wire_basis, + }, "receive": { "max": recv_max, "mean": recv_total / world_size, @@ -1119,7 +1463,8 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> component_log = (f"disp p50/p99={dp['p50']:7.1f}/{dp['p99']:7.1f} " f"comb {cp['p50']:6.1f}/{cp['p99']:6.1f} " if dp and cp else "components=unavailable ") - print(f" T={T:<5} {component_log}" + period_log = f"period={chainp['p50']:7.1f}us " if chainp else "period=n/a " + print(f" T={T:<5} {component_log}{period_log}" f"RT p50/p99={rtp['p50']:7.1f}/{rtp['p99']:7.1f}us n={len(rt)} fanout={rstats['fanout_mean']:.2f} " f"recv[min/mean/max]={recv_min}/{recv_total // world_size}/{recv_max} " f"correct={point_ok}") @@ -1181,6 +1526,11 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> }, "workload": { "cross_rank_consistent": routing_consistent, + # The ladder actually measured, plus any requested point the backend's cap excluded. + # In stdout only, a clamped ladder was invisible to anyone reading the artifact. + "ladder_measured": list(ladder), + "ladder_dropped": list(dropped), + "ladder_cap": cap, }, "measurement": { "combine_dtype": backend.combine_dtype, @@ -1189,10 +1539,17 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> "payload_unit": "token-rank", "rows": rows, "sampling": { + # The fresh-entry family. The chained family below is sampled separately and its + # counts are not derivable from these. "iterations_per_trial": args.iters, "samples_per_component": args.iters * args.trials, "trials": args.trials, "warmup_iterations": args.warmup, + # `pair_period`, `chain_floor_us` and `chain_health` sampling. Emitted because + # `sample_count` cannot be decomposed back into them: 128x4 is not 512x1. + "chain_drop": args.chain_drop, + "chain_iterations_per_trial": args.chain_iters, + "chain_trials": args.chain_trials, }, }, "implementation": { @@ -1204,6 +1561,18 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> # pick this per installed library version (flashinfer-ep does), so without it # a wheel bump silently changes the arithmetic behind `passed` with no trace. "combine_reduction": getattr(backend, "combine_reduction", "domain-fp32"), + # The library version the line above was decided FROM: without it a reader cannot tell + # a correct selection from a mis-parse. None where a backend does not report one. + "library_version": getattr(backend, "library_version", None), + # Whether `roundtrip` excludes expert-output staging. It always does now, unless the + # CX_FP8_CONSUME=dequant hatch is set; older rows carried the staging copy inside the + # chain for MoRI and FlashInfer BF16, and without this field they look identical. + "stage_excluded_from_roundtrip": bool( + getattr(backend, "stage_excluded_from_roundtrip", False) + ), + # Whether this document's rows carry the chained family. Consumers key the headline on + # presence, as for `stage_excluded_from_roundtrip`; the sweep `version` does not move. + "chained_period": True, # See EPBackend.maturity: a "candidate" row measures the library, not a deployment. "maturity": getattr(backend, "maturity", None) or "unknown", "name": backend.name, @@ -1242,10 +1611,13 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> summary_rows.append(row) def _point_summary(row): + period = row["components"]["pair_period"]["percentiles_us"] + period_summary = f" period_p50={period['p50']:.1f}us" if period else "" percentiles = row["components"]["dispatch"]["percentiles_us"] if not percentiles: - return f"T={row['tokens_per_rank']}:n/a" - return f"T={row['tokens_per_rank']}:disp_p99={percentiles['p99']:.1f}us" + return f"T={row['tokens_per_rank']}:n/a{period_summary}" + return (f"T={row['tokens_per_rank']}:disp_p99={percentiles['p99']:.1f}us" + f"{period_summary}") component_summary = " ".join(_point_summary(row) for row in summary_rows) print(f"{backend.name} ep-dispatch-combine [{args.phase}/{mode}]: " diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py index cb3c43951..766709a53 100644 --- a/experimental/CollectiveX/bench/ep_mori.py +++ b/experimental/CollectiveX/bench/ep_mori.py @@ -42,8 +42,7 @@ class MoRIBackend(EPBackend): maturity = "production" # vLLM --all2all-backend mori_*; SGLang --moe-a2a-backend mori SUPPORTED_MODES = ("normal", "low-latency") SUPPORTED_PRECISIONS = ("bf16", "fp8") - combine_needs_redispatch = True - dispatch_needs_combine_cleanup = True + requires_fresh_pair = True def __init__(self, args, rank, world_size, local_rank, device): super().__init__(args, rank, world_size, local_rank, device) @@ -54,9 +53,8 @@ def __init__(self, args, rank, world_size, local_rank, device): # argv. FP8 dispatch is caller-prequantized: MoRI's dispatch kernel keys purely # on the passed tensor dtype, so handing it an e4m3 tensor selects the FP8 # dispatch kernel with no in-kernel cast. Combine stays genuinely BF16 (quant_type - # "none"). With use_external_inp_buf False the launcher - # takes the zero-copy branch, EpCombineIntraNodeKernel_bf16_p2p; the _nop2p and - # _fp8cast variants both sit behind the external-buffer branch we never enter. + # "none"). With use_external_inp_buf True (pinned below) the launcher selects + # EpCombineIntraNodeKernel_bf16_nop2p; _p2p is the registered path, _fp8cast unreachable. self._fp8_dtype = None if self._fp8: arch = torch.cuda.get_device_properties(device).gcnArchName @@ -80,9 +78,12 @@ def __init__(self, args, rank, world_size, local_rank, device): # default; `kernel_type` kwarg omitted); scale-out EP16 uses InterNodeV1, whose # required enum member is an image-lineage check. # (kernel, generation label, (block_num, rdma_block_num, dispatch_warps, combine_warps)) + # Scale-up matches what the engines pin: vLLM and SGLang both set block_num 80, + # rdma_block_num 0 and one warp_num_per_block of 16 for dispatch and combine alike. 16 is + # also the kernel ceiling (kMaxWarpGroups 8 x kWarpsPerGroup 2). Scale-out is unchanged. kernel_name, self.kernel_generation, blocks = ( ("InterNodeV1", "inter-node-v1", (96, 64, 8, 8)) if scale_out - else ("IntraNode", "intranode", (80, 0, 16, 8)) + else ("IntraNode", "intranode", (80, 0, 16, 16)) ) if self.mode == "low-latency": # LOW-LATENCY (decode) mode: IntraNodeLL, the scale-up low-latency kernel. It is @@ -92,7 +93,9 @@ def __init__(self, args, rank, world_size, local_rank, device): # layout. Its combine keeps the plain rank-deduplicated additive sum (combine is # called with weights=None -> weight_ptr 0 in mori.ops, so the gate is NOT applied # in-kernel), identical in semantics to IntraNode/normal mode ("unweighted-rank-sum", - # the base default the harness admits for low-latency). So LL differs from the normal + # the base default the harness admits for low-latency) over the same compact + # rank-deduplicated receive (the base "token-rank" receive_layout, so the + # artifact's wire basis stays rank-deduplicated). So LL differs from the normal # IntraNode path ONLY by kernel_type (set here vs omitted) and timing; every transport # method (dispatch/stage/combine/inspect_dispatch/combine_transformed) is reused as-is. # AsyncLL (enum 4) is deliberately NOT used: it is split-phase (dispatch_recv/ @@ -105,7 +108,7 @@ def __init__(self, args, rank, world_size, local_rank, device): "is out of scope; see platform_config ll_backends)" ) kernel_name, self.kernel_generation, blocks = ( - "IntraNodeLL", "intranode-ll", (80, 0, 16, 8) + "IntraNodeLL", "intranode-ll", (80, 0, 16, 16) ) self._kernel_type = None if kernel_name != "IntraNode": @@ -118,15 +121,21 @@ def __init__(self, args, rank, world_size, local_rank, device): self._inter_node = kernel_name == "InterNodeV1" self.num_qps = 1 self.block_num, self.rdma_block_num, self.dispatch_warps, self.combine_warps = blocks - self._external_input = self._inter_node - # Registered-input MoRI copies expert output into a device-side symmetric buffer. External - # input kernels consume the dispatch output directly, so their stage is not applicable. - # Under FP8, stage also dequantizes the received fp8 payload to BF16 (device work) on - # either path, so it is a timed component regardless of the input-buffer mode. - self.stage_device_work = self._fp8 or not self._external_input + # External-input kernels consume the dispatch output directly, so stage has no copy of + # its own; under FP8 it still dequantizes the received fp8 payload to BF16, so it is a + # timed device component in that precision only. + self.stage_device_work = self._fp8 # Stash the __init__-only locals the moved create_buffer body reads back. self._gpus_per_node = gpus_per_node + def buffer_cap(self, args): + if self.mode == "low-latency": + # 256 tokens/rank, matching deepep-v2, uccl-ep and nccl-ep so every backend's + # low-latency ladder ends at the same rung. MoRI imposes no bound of its own; 256 is + # also vLLM's DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP. + return 256 + return None + def create_buffer(self, spec): args, world_size, rank = self.args, self.world_size, self.rank gpus_per_node = self._gpus_per_node @@ -151,8 +160,9 @@ def create_buffer(self, spec): f"MoRI realized {realized_qps} QPs per PE; {self.num_qps} required" ) - # MoRI preallocates one communicator buffer for the case's entire ladder. - self._cap = max(512, spec.max_tokens_per_rank) + # MoRI preallocates one communicator buffer for the case's entire ladder; 256 is the + # low-latency cap (see `buffer_cap`). Normal mode still takes the larger ladder maximum. + self._cap = max(256, spec.max_tokens_per_rank) # quant_type stays "none" for both precisions: dispatch precision is carried by # the passed tensor dtype (caller-prequantized e4m3 under FP8, BF16 otherwise), # and "none" keeps combine a genuine BF16 send. data_type is deprecated upstream @@ -172,7 +182,11 @@ def create_buffer(self, spec): "max_num_inp_token_per_rank": self._cap, "num_experts_per_rank": self.experts_per_rank, "num_experts_per_token": args.topk, - "use_external_inp_buf": self._external_input, + # External input buffer everywhere, as the engines run it. It must move together + # with `combine_warps`: MoRI's tuned tables key combine on `zero_copy`, so 16 warps + # belong to this mode and 4-8 to the registered one. methodology.md has the cost of + # mismatching them. + "use_external_inp_buf": True, "quant_type": "none", } if self._kernel_type is not None: @@ -194,14 +208,14 @@ def create_buffer(self, spec): "data_type": torch.bfloat16, "scale_dim": 0, "scale_type_size": 1, - "use_external_inp_buf": self._external_input, + "use_external_inp_buf": True, "quant_type": config_kwargs["quant_type"], } if self._inter_node: expected_config.update({ "block_num": self.block_num, "warp_num_per_block": self.dispatch_warps, - "gpu_per_node": 8, + "gpu_per_node": gpus_per_node, "rdma_block_num": 64, "num_qp_per_pe": 1, }) @@ -219,34 +233,30 @@ def semantic_payload(self, x): return x return x.to(self._fp8_dtype).to(torch.bfloat16) - def _encode_dispatch(self, x): - if not self._fp8: - return x, None - quantized = x.to(self._fp8_dtype) - return quantized, quantized.to(torch.bfloat16) - def make_problem(self, T, idx, weights, x): indices = idx.to(torch.int32) gate_weights = weights.to(torch.float32) - dispatch_x, oracle_semantic = self._encode_dispatch(x) - problem = types.SimpleNamespace( + return types.SimpleNamespace( T=T, x=x, - dispatch_x=dispatch_x, + dispatch_x=x, + oracle_x=self.semantic_payload(x), topk_idx=indices, topk_weights=gate_weights, indices=indices, weights=gate_weights, scales=torch.empty((T, 0), dtype=torch.uint8, device=self.device), ) - if oracle_semantic is not None: - problem.oracle_x = oracle_semantic - return problem def dispatch(self, p): + # Cast inside dispatch, where production pays it: vLLM and SGLang both run an aiter quant + # immediately before mori's dispatch. MoRI's cast is a single eager elementwise kernel, so + # it needs no compile. Low-latency casts here too: MoRI's IntraNodeLL takes a + # caller-prequantized tensor, unlike deepep-v2/uccl-ep whose LL kernels quantise in-kernel. + dispatch_x = p.dispatch_x.to(self._fp8_dtype) if self._fp8 else p.dispatch_x dispatch_output, dispatch_weights, _scales, dispatch_indices, recv_num = ( self.op.dispatch( - p.dispatch_x, + dispatch_x, p.weights, p.scales, p.indices, @@ -267,17 +277,13 @@ def stage(self, p, h): rows = getattr(p, "recv_tokens", None) if not isinstance(rows, int) or rows < 0 or rows > h.dispatch_output.size(0): raise RuntimeError("MoRI receive count was not validated before staging") - # FP8: dispatch delivered an e4m3 payload; dequantize it to the BF16 combine sends. + # The kernel's staging loop is bounded by `tokenIdx < totalRecvTokenNum` over + # `args.inpTokenBuf`, not by the buffer, so it never reads past `rows` and only the + # filled rows need converting. (intranode.hpp:542's P2P loop is dead on this path.) h.combine_input = ( - h.dispatch_output.to(torch.bfloat16) if self._fp8 else h.dispatch_output + h.dispatch_output[:rows].to(torch.bfloat16) if self._fp8 else h.dispatch_output ) - if self._external_input: - return None - buffer = self.op.get_registered_combine_input_buffer( - torch.bfloat16, hidden_dim=h.combine_input.size(1) - ) - buffer[:rows, :].copy_(h.combine_input[:rows, :]) - h.combine_input = buffer + return None def combine(self, p, h): combined, _weights = self.op.combine( @@ -325,12 +331,6 @@ def combine_transformed(self, p, h, transformed): rows = getattr(p, "recv_tokens", None) if not isinstance(rows, int) or rows < 0 or rows > h.combine_input.size(0): raise RuntimeError("MoRI receive count was not validated before transformed combine") - if not self._external_input: - buffer = self.op.get_registered_combine_input_buffer( - torch.bfloat16, hidden_dim=h.combine_input.size(1) - ) - buffer[:rows, :].copy_(h.combine_input[:rows, :]) - h.combine_input = buffer return self.combine(p, h) def recv_tokens(self, h): diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py index 5a616add2..8d8519321 100644 --- a/experimental/CollectiveX/bench/ep_nccl.py +++ b/experimental/CollectiveX/bench/ep_nccl.py @@ -64,6 +64,29 @@ # change can't silently truncate the id on the non-root ranks. _UNIQUE_ID_MAX_BYTES = 256 +# Low-latency receive sizing, deliberately two numbers, mirroring ep_deepep_v2: _LL_BUFFER_CAP +# sizes the pre-allocated receive (and so the transport footprint), _LL_LADDER_CAP bounds which +# token counts are measured. Separating them lets the ladder be clamped around a kernel defect +# without moving the footprint and silently re-basing the rungs that remain. +# +# The ladder sits below the buffer because nccl_ep's low_latency.cu is a port of DeepEP's +# PRE-FIX low-latency combine: in the combine recv pipeline the reduction warps read shared +# memory and then mbarrier_arrive(emptyBarriers[stageIdx]) with no fence.proxy.async.shared::cta +# between, so the producer's next TMA load can overwrite a stage while consumer reads are still +# in flight. DeepEP closed exactly this with a one-line fence in PR #642; the fence is absent +# both at our pin and at NVIDIA/nccl master, so it is unfixed upstream. +# +# Observed on gb300 EP8 BF16 at T=256: 1 failure in 5 executions, bimodal -- healthy rows give +# max relative error 0.0039, the failure gave 0.4704, with nothing between, which is a discrete +# corrupted write rather than tolerance noise. +# +# THIS CLAMP IS NOT A SAFETY BOUNDARY. The fence is missing on every combine recv; T=256 is only +# the rung with the most pipeline iterations, and the receive plane is not even full there. Lower +# rungs are LESS LIKELY to hit the race, not immune. Restore _LL_LADDER_CAP to _LL_BUFFER_CAP +# once a fixed wheel ships. +_LL_BUFFER_CAP = 256 +_LL_LADDER_CAP = 128 + class NCCLEPBackend(EPBackend): name = "nccl-ep" @@ -76,9 +99,8 @@ class NCCLEPBackend(EPBackend): SUPPORTED_MODES = ("normal", "low-latency") SUPPORTED_PRECISIONS = ("bf16",) stage_device_work = False - combine_input_attr = "combine_input_t" # this adapter's combine reads combine_input_t - combine_needs_redispatch = False - dispatch_needs_combine_cleanup = False + requires_fresh_pair = False + receive_layout = "token-rank" combine_weight_semantics = "unweighted-rank-sum" def __init__(self, args, rank, world_size, local_rank, device): @@ -100,11 +122,12 @@ def __init__(self, args, rank, world_size, local_rank, device): # unweighted rank sum — the benchmark stages the UNWEIGHTED per-expert transform # and the kernel multiplies by the gate. Same contract as deepep-v2 low-latency. self.kernel_generation = "nccl-ep-ll" + self.receive_layout = "token-expert" self.combine_weight_semantics = "weighted-kernel-sum" # NCCL EP's handle is explicitly reusable across dispatch/combine cycles (ep_test.py # cached mode redispatches and recombines on one handle), so — unlike DeepEP's legacy # low-latency Buffer — no timed component needs a fresh dispatch or a draining combine; - # both modes keep combine_needs_redispatch / dispatch_needs_combine_cleanup False. + # both modes keep requires_fresh_pair False. self._algorithm = Algorithm.LOW_LATENCY if self._ll else Algorithm.HIGH_THROUGHPUT self._layout = Layout.EXPERT_MAJOR if self._ll else Layout.FLAT # send_only=0 on every dispatch/combine (no staged execution). Handle.complete() is @@ -122,10 +145,10 @@ def __init__(self, args, rank, world_size, local_rank, device): def buffer_cap(self, args): if self._ll: - # LL pre-allocates the fixed [num_local_experts, cap*num_ranks, hidden] receive - # buffer, so cap is a hard per-rank dispatch-slot bound (same 256 as ep_deepep_v2 / - # ep_uccl low-latency; the harness clamps the decode ladder and reports drops). - return 256 + # Bounds which token counts are MEASURED. Below _LL_BUFFER_CAP today because the + # combine recv pipeline races (see the constants above); the harness reports every + # dropped rung rather than silently truncating. + return _LL_LADDER_CAP return None # ---- helpers ----------------------------------------------------------------------------- @@ -187,7 +210,10 @@ def _bootstrap_comm(self): def create_buffer(self, spec): """Bootstrap the communicator, create the EP group sized from the ladder maximum, and allocate the persistent receive/combine buffers reused across every ladder shape.""" - self.max_dispatch = spec.max_tokens_per_rank + # Sized from the BUFFER cap, not from the measured ladder, so clamping the ladder + # around the combine race does not also shrink the transport footprint -- which drives + # recv-slot memory traffic and would change what the remaining rungs measure. + self.max_dispatch = _LL_BUFFER_CAP if self._ll else spec.max_tokens_per_rank hidden = self.args.hidden self._bootstrap_comm() # max_recv_tokens_per_rank: HT requires >0 and >= max_dispatch; LL auto-derives when 0. @@ -280,6 +306,11 @@ def _ensure_handle(self, p): ) if not self._ll: h.in_weights_t = self._t(p.topk_weights) + else: + # LL applies the gate in its combine kernel, not on dispatch. Wrap the weights once + # per handle rather than per timed combine: the wrapper costs a torch resolve, an + # np.asarray and a cybind allocation, and `time_us` charges host work to the window. + h.combine_weights_t = self._t(p.topk_weights) # combined output is restored to original token order: [num_tokens, hidden]. h.out = torch.empty((p.T, self.args.hidden), dtype=torch.bfloat16, device=self.device) h.out_t = self._t(h.out) @@ -309,7 +340,7 @@ def _ensure_handle(self, p): h.handle = self._handle torch.cuda.synchronize() if not self._ll: - h.count = int(h.recv_total.item()) + self._bind_ht_recv_count(h) self._bound = h else: h.handle = self._handle @@ -317,6 +348,20 @@ def _ensure_handle(self, p): p._nccl = h return h + def _bind_ht_recv_count(self, h): + """Read HT's received-token count and pre-wrap the combine input at that size. + + Upstream sizes the combine staging copy from the tensor it is handed (`num_tokens = + x->sizes[0]`), not from the group's buffer, so handing it the whole ladder-max plane put a + rung-independent floor under HT combine -- ~470-1295us on a prefill leg (ladder max 8192). + Slicing is a free leading-dim view and matches upstream's own ep_test. Both callers are + untimed (handle creation and rebind), so the `.item()` read never lands in a window. + """ + h.count = int(h.recv_total.item()) + # A rank that received nothing still needs a non-empty tensor for the shape checks; the + # routing map decides what combine reads, so the extra row cannot reach the output. + h.combine_in_t = self._t(self._recv_x[: max(h.count, 1)]) + def _rebind(self, h): """Point the single handle at h's routing (collective; untimed callers only). @@ -332,7 +377,7 @@ def _rebind(self, h): ) torch.cuda.synchronize() if not self._ll: - h.count = int(h.recv_total.item()) + self._bind_ht_recv_count(h) self._bound = h # ---- transport contract ------------------------------------------------------------------ @@ -375,8 +420,10 @@ def dispatch(self, p): def stage(self, p, h): # BF16 combine input is the received buffer itself; no device work (value correctness - # is exercised only through the oracle's combine_transformed path). - h.combine_input_t = self._recv_x_t + # is exercised only through the oracle's combine_transformed path). LL needs the full + # padded plane, HT only the received rows (see `_bind_ht_recv_count`). + # Still an nccl.ep tensor wrapper, not a torch tensor; shared code passes it through. + h.combine_input = self._recv_x_t if self._ll else h.combine_in_t def combine(self, p, h): stream = self._stream() @@ -384,8 +431,8 @@ def combine(self, p, h): # Weighted LL combine: the kernel multiplies each expert contribution by the # source token's gate (CombineOutputs.topk_weights) before the FP32 accumulation. h.handle.combine( - CombineInputs(tokens=h.combine_input_t), - CombineOutputs(tokens=h.out_t, topk_weights=self._t(p.topk_weights)), + CombineInputs(tokens=h.combine_input), + CombineOutputs(tokens=h.out_t, topk_weights=h.combine_weights_t), config=self._combine_cfg, stream=stream, ) @@ -393,7 +440,7 @@ def combine(self, p, h): # Unweighted HT combine (FWD forbids input weights): sums the per-token expert # aggregates back to each token's home rank, restored to original order. h.handle.combine( - CombineInputs(tokens=h.combine_input_t), + CombineInputs(tokens=h.combine_input), CombineOutputs(tokens=h.out_t), config=self._combine_cfg, stream=stream, @@ -469,7 +516,7 @@ def _ll_combine_transformed(self, p, h, transformed): stream = self._stream() h.handle.combine( CombineInputs(tokens=self._t(combine_buf)), - CombineOutputs(tokens=h.out_t, topk_weights=self._t(p.topk_weights)), + CombineOutputs(tokens=h.out_t, topk_weights=h.combine_weights_t), config=self._combine_cfg, stream=stream, ) @@ -489,7 +536,8 @@ def combine_transformed(self, p, h, transformed): self._recv_x[: transformed.shape[0]].copy_(transformed.to(self._recv_x.dtype)) stream = self._stream() h.handle.combine( - CombineInputs(tokens=self._recv_x_t), + # Same sliced input the timed path uses, so the two cannot diverge in shape. + CombineInputs(tokens=h.combine_in_t), CombineOutputs(tokens=h.out_t), config=self._combine_cfg, stream=stream, diff --git a/experimental/CollectiveX/bench/ep_uccl.py b/experimental/CollectiveX/bench/ep_uccl.py index 1a61821d0..76eefb401 100644 --- a/experimental/CollectiveX/bench/ep_uccl.py +++ b/experimental/CollectiveX/bench/ep_uccl.py @@ -134,8 +134,8 @@ class UCCLEPBackend(EPBackend): SUPPORTED_MODES = ("normal", "low-latency") SUPPORTED_PRECISIONS = ("bf16", "fp8") stage_device_work = False - combine_needs_redispatch = False - dispatch_needs_combine_cleanup = False + requires_fresh_pair = False + receive_layout = "token-rank" combine_weight_semantics = "unweighted-rank-sum" def __init__(self, args, rank, world_size, local_rank, device): @@ -155,15 +155,18 @@ def __init__(self, args, rank, world_size, local_rank, device): ) self.dispatch_value_bytes = 1 self.dispatch_scale_bytes_per_copy = ((args.hidden + 127) // 128) * 4 + # Normal/HT quantises inside the timed dispatch with the compiled form; low-latency + # keeps the eager helper, whose bits its in-kernel cast matches. See fused_quantize. + self._quant = self.fused_quantize(per_token_cast_to_fp8) if self.mode == "low-latency": # Legacy low-latency decode path: a distinct kernel family whose combine multiplies # by the gate at the source (weighted), not an unweighted rank sum. LL result tensors # are double-buffered and single-use per dispatch, so every timed combine needs a # fresh dispatch and every timed dispatch must be drained by its combine. self.kernel_generation = "uccl-legacy-buffer-ll" + self.receive_layout = "token-expert" self.combine_weight_semantics = "weighted-kernel-sum" - self.combine_needs_redispatch = True - self.dispatch_needs_combine_cleanup = True + self.requires_fresh_pair = True def buffer_cap(self, args): if self.mode == "low-latency": @@ -273,20 +276,13 @@ def _topk_idx_dtype(self): def semantic_payload(self, x): if not self._fp8: return x - return per_token_cast_back(*per_token_cast_to_fp8(x)) + # Same callable the wire uses, so sender and oracle cannot disagree by construction. + return per_token_cast_back(*self._quant(x)) - def _encode_dispatch(self, x): - if not self._fp8: - return x, None - if self.mode == "low-latency": - # low_latency_dispatch takes BF16 x and casts to e4m3 inside the kernel, so send x - # unquantized; expose the host round-trip as the oracle semantic. - return x, per_token_cast_back(*per_token_cast_to_fp8(x)) - fp8, scales = per_token_cast_to_fp8(x) - # Column-major (TMA-compatible) scale layout the dispatch kernel expects, matching UCCL's - # own bench (`scales.T.contiguous().T`) and the LL scale-contiguity note below. - quantized = (fp8, scales.T.contiguous().T) - return quantized, per_token_cast_back(fp8, scales) + def _validate_quantizer(self, x): + # Low-latency keeps the eager quantize; nothing to cross-check there. + if self._fp8 and self.mode != "low-latency": + self.assert_quantize_identity(per_token_cast_to_fp8, self._quant, x) def _ll_recv_bf16(self, recv_x): """The padded per-expert receive as BF16 [num_local_experts, cap*num_ranks, hidden]. @@ -325,8 +321,16 @@ def dispatch(self, p): # it through so the same call serves both scopes. (num_tokens_per_rank, num_tokens_per_rdma_rank, num_tokens_per_expert, is_token_in_rank, _) = self.buffer.get_dispatch_layout(p.topk_idx, self.args.experts) + # Quantise here, not in make_problem: production runs one fused bf16->fp8 kernel per forward + # pass right before this collective. The scales need UCCL's column-major (TMA-compatible) + # layout, which production's kernel emits directly -- so timing the transpose over-states + # by one small copy. + dispatch_x = p.dispatch_x + if self._fp8: + fp8, scales = self._quant(dispatch_x) + dispatch_x = (fp8, scales.T.contiguous().T) recv_x, recv_topk_idx, recv_topk_weights, _counts, handle, _event = self.buffer.dispatch( - x=p.dispatch_x, + x=dispatch_x, num_tokens_per_rank=num_tokens_per_rank, num_tokens_per_rdma_rank=num_tokens_per_rdma_rank, is_token_in_rank=is_token_in_rank, diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json index 52de60ba9..506538f4b 100644 --- a/experimental/CollectiveX/configs/platform_config.json +++ b/experimental/CollectiveX/configs/platform_config.json @@ -17,7 +17,7 @@ "partition": "hpc-gpu-1", "account": "customer", "squash_dir": "/mnt/nfs/sa-shared/cx-squash", - "exclude_nodes": "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + "exclude_nodes": "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-2,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" }, "network": { "socket_ifname": "eth0", @@ -45,7 +45,7 @@ "rdma_devices": "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7" } }, - "b200-dgxc": { + "b200-nscale": { "arch": "sm100", "product": "b200", "image": "lmsysorg/sglang:v0.5.11-cu130", @@ -55,17 +55,16 @@ "scale_up_transport": "nvlink", "launcher": "single-slurm", "backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8]}, - "ll_backends": {"deepep-v2": [8], "uccl-ep": [8], "nccl-ep": [8]}, - "fabric": {"nic": "ConnectX-7 400GbE", "switch": "Whitebox Tomahawk3 leaf + Tomahawk4 (RoCE)"}, + "ll_backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8]}, + "fabric": {"nic": "8x ConnectX-7-class 400G InfiniBand rails (bare metal, gdrdrv)", "switch": "InfiniBand (model not surveyed)"}, "operator": { - "partition": "gpu-2", + "partition": "batch_1", "account": "benchmark", - "qos": "gpu-2_qos", - "squash_dir": "/home/sa-shared/containers" + "squash_dir": "/data/home/sa-shared/sqsh" }, "network": { - "rdma_devices": "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7", - "ib_gid_index": "3" + "socket_ifname": "bond0", + "rdma_devices": "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11" } }, "b300": { @@ -84,7 +83,8 @@ "partition": "batch_1", "account": "benchmark", "qos": "batch_1_qos", - "squash_dir": "/data/home/sa-shared/sqsh" + "squash_dir": "/data/home/sa-shared/sqsh", + "exclude_nodes": "b300-018" }, "network": { "socket_ifname": "bond0", @@ -103,7 +103,7 @@ "scale_up_transport": "mnnvl", "launcher": "gb-nv", "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]}, - "ll_backends": {"nccl-ep": [8]}, + "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8]}, "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"}, "operator": { "partition": "batch", @@ -123,14 +123,15 @@ "scale_up_transport": "mnnvl", "launcher": "gb-nv", "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]}, - "ll_backends": {"nccl-ep": [8]}, + "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8]}, "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"}, "operator": { "partition": "batch_1", "account": "benchmark", "qos": "batch_1_qos", "squash_dir": "/data/home/sa-shared/collectivex/containers", - "enroot_cache_path": "/data/home/sa-shared/collectivex/enroot-cache" + "enroot_cache_path": "/data/home/sa-shared/collectivex/enroot-cache", + "exclude_nodes": "im-gb300-r01-c003,im-gb300-r01-c005" } }, "mi325x-tw": { diff --git a/experimental/CollectiveX/configs/sweep.json b/experimental/CollectiveX/configs/sweep.json index 22c873e9f..212090224 100644 --- a/experimental/CollectiveX/configs/sweep.json +++ b/experimental/CollectiveX/configs/sweep.json @@ -11,7 +11,10 @@ "timing": { "iters_per_trial": 8, "trials_per_point": 256, - "warmup_iters_per_trial": 32 + "warmup_iters_per_trial": 32, + "chain_iters_per_trial": 128, + "chain_trials_per_point": 4, + "chain_drop": 16 }, "workload": { "name": "deepseek-v3", diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md index 8108c3fd4..50acdddd9 100644 --- a/experimental/CollectiveX/docs/methodology.md +++ b/experimental/CollectiveX/docs/methodology.md @@ -22,12 +22,68 @@ It does not predict serving throughput without a separate correlation study. The implemented workload is `deepseek-v3`: hidden 7168, top-k 8, 256 routed experts, packed placement, and one pinned fixed resource profile per backend/topology. Combine is always BF16; dispatch precision is a swept dimension — a BF16 control and, on the backends whose FP8 dispatch is -supported upstream (DeepEP V2, MoRI, UCCL-EP), an FP8 dispatch (`bf16`, `fp8`), +supported upstream (DeepEP V2, MoRI, UCCL-EP, FlashInfer EP), an FP8 dispatch (`bf16`, `fp8`), caller-prequantized in `normal` mode (the `low-latency` kernels quantize FP8 internally from BF16 on -DeepEP and UCCL-EP, and stay caller-prequantized on MoRI). NCCL EP is BF16-only this release, so its -cells carry the control alone; the per-backend precision set lives in `sweep_matrix.py`'s -`BACKEND_PRECISIONS` and a backend never emits a case for a precision it does not support. -`normal`-mode cases use the +DeepEP and UCCL-EP, and stay caller-prequantized on MoRI). That caller-side quantize is charged +**inside the measured dispatch**, because a production forward pass pays it on the critical path. It +is one fused kernel on DeepEP V2, UCCL-EP and FlashInfer EP, guarded bitwise against its eager +reference; MoRI's is a plain dtype cast and needs neither. So an FP8 `normal` dispatch covers +quantize-plus-transport while its BF16 control covers transport alone, and is not comparable to a row +measured before that change. The sweep `version` deliberately stays 1 across it, so +`implementation.stage_excluded_from_roundtrip` and the presence of a `stage` component are the +discriminators for THAT change. + +Those two fields do not separate every generation, and it is worth being exact about which ones +they miss, because the version tag will not help. A same-cell comparison across the durable store +puts numbers on it: the FP8 quantize charge moved dispatch by +64% to +110% (median, small T); the +staging hoist widening moved FlashInfer roundtrip −46% and MoRI BF16 −23%; MoRI's move to the +external input buffer with 16 warps moved FP8 combine +33% to +65% on gfx950; sizing the NCCL EP +HT combine to its receive count moved combine −10% to −41%; and the two-pass chain moved +`pair_period` −5% to −42% everywhere. Of those, only the first two are keyed by the fields above. + +One more key, for the rest. **`chain_health.interpair_gap_us` present** marks the two-pass chain: +rows from the single six-events chain carry `chain_health` with `pair_spread_us` alone, and their +`pair_period` is inflated by a host constant rather than being a different-but-valid quantity — so +treat those rows as defective, not merely older. For the MoRI buffer-mode change and the NCCL EP +combine sizing there is **no per-row discriminator at all** — `kernel_generation` reads the same on +both sides — so pre-change rows for those two backends cannot be separated from post-change rows by +any field, and must be excluded from the store rather than keyed around. + +If the sweep version is ever bumped, **skip 2 and go to 3.** A short-lived bump to 2 during this +branch's development left stored documents tagged `version: 2` that are currently invisible only +because the frontend reader accepts `[1]`. Publishing support for 2 would silently resurface that +mid-development churn as valid data. + +The `fp8_consume` derivation, in full, since the code now points here for it. `dequant` is a +verification hatch rather than a second metric — never a sweep axis, never a default — because the +mismatched-config cost is derivable from what every run already emits: + + dequant roundtrip ~= roundtrip + stage (+2.4% .. -0.1%, b200 LL fp8 ladder) + +slightly high because chaining amortises launch overhead (median `rt/(d+s+c)` = 0.93 across the +corpus). The reverse does **not** hold: reconstructing native as `dequant - stage` errs by -11.6% at +T=1, -5% at T=64, and converges only by T=256 — worst precisely in the decode regime the headline +reports. So measure native and derive dequant, never the other way round. The hatch reproduces +historical deepep-v2/uccl-ep numbers for regression checks (302.0µs against 302.5µs in run +30177021271 at T=1); it does not reproduce MoRI fp8, whose stage now casts only the rows dispatch +filled, nor any pre-hoist BF16 roundtrip. + +Read that charge as a **fixed per-call cost, not a payload-proportional one**. On DeepEP V2 decode, +FP8 dispatch p50 minus its BF16 control is 65us on h100, 59us on b200, 27us on b300 and 57us on +gb300 at T=1, flat within a microsecond or two through T=64, then decaying; by T=512 it is slightly +negative on h100 and b300, where halved payload bytes more than repay it. FlashInfer EP carries +~107us on gb300 (its own codec, and a fourth dispatch payload under FP8). At T=1 the FP8 path moves +*fewer* bytes than BF16, so this is per-call work, not transport, and it exceeds the fused quantize's +own device time (1.5-3.6us per SKU) by more than an order of magnitude: the timing window has no host +sync before its start event (see below), so a near-idle stream at T=1 lets host-side launch cost land +inside it. Compiling the quantize reduces this rather than causing it — an eager quantize measures +33-39us worse per decode dispatch on h100 through the same window — but production issues one custom +quantize op into an already-busy stream, so the small-T end of an FP8 `normal` row is the least +production-representative number the suite emits. Compare FP8 and BF16 at the top of the ladder. +`low-latency` rows are unaffected: those kernels quantize internally or take pre-quantized input by +API contract. NCCL EP is BF16-only this release, so its cells carry the control alone; the +per-backend precision set lives in `sweep_matrix.py`'s `BACKEND_PRECISIONS` and a backend never +emits a case for a precision it does not support. `normal`-mode cases use the `layout-and-dispatch-v1` semantics; `low-latency` cases use each backend's decode-kernel semantics (detailed below). @@ -35,6 +91,22 @@ cells carry the control alone; the per-backend precision set lives in `sweep_mat T=1..512 powers of two and prefill T=1024..8192 powers of two. Ladders are model-specific and live with the workload in `configs/sweep.json`. +A backend may clamp the ladder below that, and every clamped point is reported in the artifact +rather than dropped silently — `workload.ladder_measured`, `ladder_dropped` and `ladder_cap` +record what ran and what did not. DeepEP V2 in `low-latency` mode pre-allocates a fixed receive, +so its ladder cannot exceed that buffer; both are 256, so the decode ladder runs to its full +extent and only the 512 point is dropped. + +That clamp was briefly load-bearing. DeepEP's low-latency combine corrupted the 256 rung on every +Blackwell SKU — B200, GB200 and GB300, EP8 and EP16, both precisions, MNNVL and RDMA alike — while +Hopper stayed clean, stochastically at roughly 1.5-3.3% per invocation, surfacing as one wrong token +row whose norm still matched to 4 significant figures. Upstream PR #642 fixed it with a CTA-scope +fence retiring the combine consumer's shared-memory reads before its staging buffer is recycled; the +commit we pinned was a pre-merge branch head that predated it, so we clamped the measured ladder to +128 until the pin moved to upstream main. The receive is sized from a constant rather than from +`max(ladder)` so clamping cannot change the footprint, which drives both the transport's memory +traffic and the FP8 dequant volume. + `sweep_matrix.py` materializes the requested SKUs, backends, EP sizes, and token ladders into a matrix document, then extracts strict per-shard controls. `--only-sku`, `--exclude-skus`, `--ep-sizes`, and `--precisions` select a subset; a subset produces a smaller matrix, not a @@ -47,17 +119,55 @@ case count. | MI300X/MI325X/MI355X | 1x8 XGMI, scale-up | 2x8 XGMI + RDMA, scale-out | | GB200/GB300 | 2x4 MNNVL, scale-up | 4x4 MNNVL, scale-up | +**A virtualized pool can make a scale-out row measure the hypervisor rather than the fabric.** +h200-dgxc EP16 pays roughly three times the cross-node cost of b300 or h100 on identical topology +and identical traffic, while its EP8 rows are correct — the deficit is confined to the hop. It +sustains ~34 GB/s per node against a nominal 8x400G (~4.2 GB/s per GPU-NIC pair) where bare-metal +h100 reaches wire rate. Reordering the NIC-PE mapping to pair each rank with its socket-local NIC +changed nothing (478µs against a 480µs baseline), which rules the selector out and points at the +GDR path being degraded wholesale inside the guest. The retired b200-dgxc pool showed the same +shape. Treat EP16 rows from a virtualized pool as a lower bound on the hardware until the host's +ACS/IOMMU configuration is confirmed. + Physical host count does not define scope. Both GB cells remain inside one 72-GPU MNNVL scale-up domain. Unsupported combinations are explicitly classified in the matrix, not silently skipped coverage. DeepEP V2 is the -`ElasticBuffer` introduced by PR #605, pinned with upstream PR #630's minimal pure-scale-up fix and -the exact upstream PR #640 library matcher that excludes NCCL shared-memory mappings. Scale-up cases +`ElasticBuffer` introduced by PR #605, pinned at upstream main, which carries that PR plus #630's +minimal pure-scale-up fix, the #640 library matcher that excludes NCCL shared-memory mappings, and +the #642 low-latency combine fence. Scale-up cases request NCCL Device API LSA and fail closed unless the realized LSA team covers the full EP world. x86 EP16 scale-out uses the hybrid path with GIN and requires two logical scale-out domains represented by two physical RDMA ranks, with eight scale-up ranks per domain. GB EP16 remains MNNVL scale-up and uses LSA. MoRI EP8 uses the direct IntraNode kernel on every CDNA SKU; its EP16 InterNodeV1 path is -configured but unsupported (transport-layer combine corruption, ROCm/mori#475) and never dispatched. UCCL-EP is a drop-in, API-identical DeepEP replacement that keeps the legacy `Buffer` +configured but unsupported (transport-layer combine corruption, ROCm/mori#475) and never dispatched. +MoRI runs under its MANUAL launch mode with a pinned launch config, because that is what the engines +run: neither vLLM nor SGLang sets `MORI_EP_LAUNCH_CONFIG_MODE`, and both pin block_num 80, +rdma_block_num 0, and `warp_num_per_block` 16 for the intra-node kernel, on dispatch and combine +alike (neither passes a per-call override), with an external input buffer — MoRI's default, which +SGLang sets explicitly. The two are pinned together deliberately: MoRI's tuning tables key combine on +`zero_copy`, selecting roughly 16 warps for external input against 4-8 for a registered buffer, and +the mismatch is measurable in both directions. On MI300X, MI325X and MI355X, +**in registered-buffer mode** 16 warps costs +13-18% combine at T=128 and +61-78% at T=512 against 8; +**in the external-input mode the engines actually run** the same 16 warps *wins* — 14-19% at T=128, +26-27% at T=256, and 9-14% at every prefill rung including T=8192. Below T=32 it gives up 0.2-2.5us, +the only range where 8 is ahead; all arms were correct at every rung. With an external input buffer +the kernel does its own staging copy, bounded by the receive count, so BF16 rows hand over the +dispatch output unchanged and declare `stage` as an explicit unavailable marker (null percentiles, +zero samples); FP8 rows still stage for real, to dequantize the received payload. These numbers +describe the engine-integrated configuration, not MoRI's peak: its shipped tuning tables reach a +faster combine with per-shape block and warp counts no engine selects, and AUTO would not reproduce +them uniformly — gfx950 ships no IntraNodeLL combine table and no BF16 rule for normal-mode +IntraNode dispatch, so AUTO defaults exactly those two (coupling the result to whichever MoRI +revision is pinned) while tuning the other two, which is not one number about the hardware. How far +off peak is arch-dependent: across buffer modes on MI355X, with +the registered mode's excluded BF16 stage added back, a registered buffer at 8 warps is still 15% +faster at T=512 decode and 8% at T=8192 prefill than the shipped pairing, but that did not reproduce +on gfx942, so treat 0-15% as the honest range for a configuration no engine runs. The low-latency +arm has no engine-integrated configuration to match at all: SGLang's low-latency path pins `AsyncLL` +at 8 warps while this suite uses `IntraNodeLL` (`AsyncLL` is split-phase and fails silently under a +single-call harness), so its launch config is inherited from the normal-mode tuple by choice rather +than by precedent. UCCL-EP is a drop-in, API-identical DeepEP replacement that keeps the legacy `Buffer` `dispatch`/`combine` (unweighted rank-sum) but routes it over CPU-proxy GPUDirect RDMA on plain `libibverbs` — no NVSHMEM/IBGDA — with software message ordering, atomics, and flow control; its scale-up is single-node `cudaIpc` over NVLink/XGMI (so the scale-up domain is one physical node, @@ -68,7 +178,7 @@ unweighted rank-sum combine match `layout-and-dispatch-v1` exactly, so the same NVIDIA-only and CUDA 13 only, and runs EP8 scale-up on H100/H200/B200/B300 plus EP8 and EP16 on GB200/GB300, where EP16 stays inside the MNNVL scale-up domain; x86 EP16 scale-out is an unsupported coverage row, its cross-node GIN path faulting inside `nccl_ep.cc` identically on RoCE and IB across -four SKUs — a GDAKI limit, not a fabric-selection one. FlashInfer EP is TensorRT-LLM's one-sided MNNVL `MoeAlltoAll`, in which each rank writes tokens directly into its peers' workspace windows and combine reads them back, so there is no send/recv pairing and no NVSHMEM; it is GB200/GB300-only for that reason, and runs EP8 and EP16 inside the MNNVL scale-up domain. Its combine is the one place a backend's accumulator precision changes the expectation rather than the tolerance: through 0.6.15 the kernel holds its top-k accumulators in the payload dtype and reduces them with a hand-unrolled pairwise tree, so every level rounds to BF16, and the oracle reproduces that tree exactly rather than loosening the gate to absorb it (0.6.16 rewrote the accumulator to FP32; the adapter reads the installed version and picks the matching model). Those throughput kernels run across the full token ladder in the `normal` mode. +four SKUs — a GDAKI limit, not a fabric-selection one. FlashInfer EP is TensorRT-LLM's one-sided MNNVL `MoeAlltoAll`, in which each rank writes tokens directly into its peers' workspace windows and combine reads them back, so there is no send/recv pairing and no NVSHMEM; it is GB200/GB300-only for that reason, and runs EP8 and EP16 inside the MNNVL scale-up domain. Its combine is the one place a backend's accumulator precision changes the expectation rather than the tolerance: through 0.6.15 the kernel holds its top-k accumulators in the payload dtype and reduces them with a hand-unrolled pairwise tree, so every level rounds to BF16, and the oracle reproduces that tree exactly rather than loosening the gate to absorb it (0.6.16 rewrote the accumulator to FP32; the adapter reads the installed version and picks the matching model). Those throughput kernels run across the full token ladder in the `normal` mode. Its FP8 dispatch is the one (backend, precision) pair here that is realizable but off every deployed path — vLLM accepts only nvfp4/mxfp8/bf16 on this transport — so `sweep_matrix.py`'s `OFF_PATH_PRECISIONS` keeps it out of the default matrix and a production sweep measures only configurations an engine can select. Naming the precision explicitly (`--precisions fp8`) opts it back in for transport comparison against DeepEP V2/UCCL-EP at matching bytes and block size: the one place a precision filter ADDS rows rather than only removing them. A second `low-latency` mode adds each backend's decode-optimized kernel family. On DeepEP it drives the legacy `deep_ep.Buffer` low-latency decode kernels (`low_latency_dispatch`/`low_latency_combine`), @@ -83,7 +193,9 @@ rank-sum combine as the throughput `IntraNode` kernel, so it differs only by ker does not fit the single-call dispatch/combine contract). Low latency is a decode-phase-only addition whose runnable set is narrower than and distinct from the throughput kernels', so it is enabled cell-by-cell from the registry's `ll_backends` map rather than assumed wherever `normal` runs; it is -currently enabled for DeepEP V2 EP8 on H100/H200/B200, MoRI +currently enabled for DeepEP V2 at EP8 on H100/H200 and at EP8 and EP16 on B200 (the nscale +bare-metal pool: IBGDA over native IB rails with `/dev/gdrdrv`, which an x86 low-latency scale-out +needs and no virtualized pool has) and on GB200/GB300 (EP16 inside the MNNVL scale-up domain), MoRI EP8 on MI300X/MI325X/MI355X, and UCCL-EP EP8 on H100/H200/B200 only (the legacy `Buffer` low-latency kernels; at EP8 these run `cudaIpc` over NVLink, not the CPU-proxy RDMA path, because the adapter passes `is_intranode` and UCCL then never starts its proxies. The AMD SKUs drop LL: upstream raised `kNumMaxTopK` 9 -> 16 @@ -91,8 +203,8 @@ six days before our pin, and the resulting host assert cannot hold on AMD's 16 w `LOW_LATENCY` algorithm is the DeepEP-derived decode path, EXPERT_MAJOR receive with a source-side weighted-kernel-sum combine. Those rows were dropped while every LL leg wedged on stale peer signals ([NVIDIA/nccl#2303](https://github.com/NVIDIA/nccl/issues/2303)) and restored once the single-handle -adapter removed the aliasing that caused it. B300, GB200 and GB300 carry NCCL EP as their only -low-latency row, and it is a `candidate` transport, so those three SKUs publish no production decode +adapter removed the aliasing that caused it. B300 carries NCCL EP as its only +low-latency row, and it is a `candidate` transport, so that SKU publishes no production decode coverage. Whether a given SKU/backend/EP/mode cell is attempted is a capability fact; whether it succeeded is decided only by the emitted artifact. @@ -115,22 +227,188 @@ Adapters may not generate routing or reinterpret one quantity as the other. Normal mode uses `layout-and-dispatch-v1`: dispatch timing includes layout plus communication, and combine returns activation payload through an unweighted rank-sum path. Expert-output staging is -outside isolated combine timing and inside the measured paired roundtrip. Each component declares +outside isolated combine timing AND outside the measured paired roundtrip, so `roundtrip` means +dispatch then combine — the transport — in every row, and staging is reported as its own `stage` +component wherever it does device work. The `CX_FP8_CONSUME=dequant` verification hatch is the one +exception, putting the conversion back inside the chain on purpose. + +Under FP8, treat `stage` as **harness scaffolding rather than a phase a serving stack has**: it +converts the received FP8 payload to the BF16 combine sends, work production never does separately — +the FP8 lands in the expert GEMM, which reads FP8 operands natively and emits the BF16 combine +receives. This suite measures the collective, not the layer, so `stage` stands in for that GEMM, +which is why it is excluded from `roundtrip` and why **`stage` must not be summed into a total or +compared between backends**: each adapter converts a different amount. DeepEP V2 and UCCL-EP convert +only the received rows in `normal` mode but the whole padded plane in `low-latency`, where the +receive buffer is `[experts, cap * ranks, hidden]` regardless of token count; MoRI converts only the +received rows; FlashInfer only the filled slots. The one production path that *does* pay a separate +materialised dequant is a quant-format mismatch fallback (vLLM dequantises when `block_k` disagrees +with DeepEP's block size), which `CX_FP8_CONSUME=dequant` models; it is not the default because it +is not the fast path. + +Read `implementation.stage_excluded_from_roundtrip` as "there was device-work staging and it was +hoisted out of the chain", not as "this row's roundtrip is stage-free". It is gated on whether the +backend's `stage()` does device work at all, so `false` covers two unrelated situations that the +`stage` component separates: **absent** means the backend has nothing to stage (a bare pointer +assignment, as for NCCL EP and every BF16 row that hands the receive buffer straight to combine), +**present alongside `false`** means the `dequant` hatch put the conversion back inside the chain. +Reading `false` alone as "roundtrip includes staging" subtracts a cost the row never paid. Each component declares availability, origin, and sample count. A paired-only API reports null isolated components. -`isolated_sum` is derived. The artifact records the mode so a reader can keep distinct measurement -contracts separate. +`isolated_sum` is derived. + +Headline latency is the **chained pair period** (`components.pair_period`, defined under Chained +Pair Period below) for every row that carries one, and the p99 of the per-iteration cross-rank MAX +of `roundtrip` for rows measured before that field existed. The flip shipped **held** while the +six-events-per-pair chain described below — whose inner records inflated small-T periods +fleet-wide — was replaced by the two-pass chain, and was released on 2026-08-06 once the b200, h200 +and gb200 hand references were confirmed against two-pass fleet artifacts (runs 31092783122 and +31089556516). Both `p50` and `p99` are +emitted either way and `summarize.py` prints both. MAX is the fresh-entry family's reduction because +a layer is not finished until its slowest rank is, so MAX is the completion cost, and it charges +inter-rank entry stagger to whichever component the ranks entered unevenly. That stagger depends on +the code path AND the precision, not only on the fleet: on identical h200 low-latency decode cells +the per-iteration spread is ~9.3 us for deepep-v2 and uccl-ep at BF16 (they share the legacy +`Buffer` path) against ~2.6 us for nccl-ep, and collapses to ~2.8 us for those same two under FP8, +where in-kernel quantisation makes the heavier dispatch self-align the ranks. The term is not +subtractable in any principled way, so MAX alone taxes some rows more than others. + +Every row therefore also carries `cross_rank_min_us` (the same iterations reduced with MIN — the +skew-excluded floor) and `cross_rank_spread_us` (per-iteration MAX minus MIN). Read MAX and MIN as a +bracket: two cells whose MAX gap is smaller than the larger contender's spread are not separated by +the data. Rank on roundtrip p50 and call a winner only where MAX and MIN agree on the ordering. Do +not rank on p99 of MAX for multi-node decode cells, where it is dominated by worst-rank stalls +rather than transport — p99 of MIN is the synchronized-cost tail beside it. The isolated components +inherit the preceding operation's per-rank exit stagger, so treat them as residual-wait diagnostics +rather than per-operation costs; the paired roundtrip is the comparable quantity. + +### Chained Pair Period + +Everything above measures **fresh entry**: drained around each timed window, so every +sample starts from an idle pipeline and the ranks re-stagger before each one. A decode loop never +stops, and what it pays per MoE layer is the pipeline's steady-state **period**. Every row therefore +carries the chained family, measured by `benchmark_chain`: dispatch→combine pairs issued +back-to-back with CUDA events enqueued on-stream and **no host synchronization inside the loop**, +the first `chain_drop` (16) pairs discarded as pipeline fill, pooled over `chain_trials` (4) per +point on the same rotated ladder order as the rest of Pass 2. The pairing is exactly +`run_roundtrip`'s — dispatch, the staged combine input (or an inline `stage` under the +`CX_FP8_CONSUME=dequant` hatch), then combine — so paired-API backends stay in contract and +`pair_period` excludes expert-output staging on the same rule `roundtrip` does. + +Each trial runs **two sibling chains** of `chain_iters` (128) pairs, because the statistics must +not carry the instrumentation that collects them. The first version ran ONE chain with six +`record()` calls per pair; wherever the device drains faster than the host enqueues (the bottom +of the ladder), every event executes as issued, the pair window degenerates to host elapsed time, +and the four inner records plus glue were charged into the published period. The fleet exposed it +before a profiler did: period minus floor-sum sat at a roughly T-independent 10–30µs on every +vendor and fabric at once — a host constant, not transport — inflating T=1 periods by 20–38%. So a +**floors chain** runs first carrying only the four op-window events, and a **period chain** runs +second carrying only the outer pair events, nothing between its two collectives: both records' host +cost lands in the inter-pair gap, outside the window, so the period carries what an uninstrumented +caller pays. (An eager-mode launch floor remains, as for any eager caller; a CUDA-graphs decode loop +pays less host per pair than any eager harness can.) + +The two chains publish five statistics, and only five: + +- `components.pair_period` (origin `chained-median`) — the per-pair period from the period chain, + reduced across ranks by MEDIAN. MAX is right for a drained component (a layer finishes with its + slowest rank), but the period is a **rate**: the collectives phase-lock every rank into one + cadence, and a MAX would publish whichever rank hiccuped as the pipeline's speed. +- `chain_floor_us.dispatch` / `.combine` (origin `chained-cross-rank-min`) — each op's window from + the floors chain, reduced across ranks by MIN: the last rank into a collective waited least, so + its window is the op's floor, and it tracks profiler kernel time to ~10% — a free Kineto + substitute. Read floor-vs-period as transport share, not an identity that must close to zero — + `period − Σfloors` is a real quantity with a meaning in each sign, and neither sign is an error: + - **Positive** is the per-pair inter-rank wait that the MIN deliberately strips. Where a backend + is synchronization-dominated it rivals the floor sum and sits *flat in T* — gb200 + flashinfer-ep normal decode (run 31089556516) holds ~70µs bf16 / ~100µs fp8 at every rung, + vanishing by T=512 and in prefill as the floors grow into it. That is `period = max(sync + budget, work)`, not instrumentation. It is invisible to `pair_spread_us` (2.7–8.5µs against + 62–107µs gaps) because the period is conserved while the wait migrates between op windows — + the same steady-state stagger that bans chained per-op medians below. + - **Negative** is the floors chain's own four-records-per-pair host cost (~10–12µs) inflating + *its* windows wherever the device outruns the host: the very effect the two-chain split + evicted from the period, still present and harmless in the floors (gb200/h200 fp8 LL, run + 31089556516). It is not overlap, and it is not cross-chain settling. + + Do not read a large positive residual as the six-events defect returning. That defect was a + roughly T-independent host constant appearing on *every vendor and fabric at once*, and it lands + in `interpair_gap_us`; a sync-dominated residual is backend-specific and leaves the gap small. +- `chain_health.pair_spread_us` — per-iteration cross-rank max-minus-min of the pair: the *proof* + the median means anything. Large next to `pair_period` means a paced or slow rank, and the + point should not be read as a steady-state period at all. +- `chain_health.interpair_gap_us` — start-to-start median minus pair-window median, once per + trial: the per-pair cost OUTSIDE the published window (the harness's own two `record()` calls + plus any inter-pair stall), and the in-artifact regression guard against the six-events defect + above. +- `chain_health.settle_drift_us` — late-half minus early-half period median, once per trial, + signed, cross-rank max-magnitude. `chain_drop` *assumes* the chain settled before the kept + pairs; this is the proof, and an unconverged chain (or a device clocking down mid-chain) + publishes its drift instead of a clean-looking period. + +**Chained per-op medians and p99s are never published.** Without a host sync the ranks arrive at +each collective at slightly different times, and the resulting wait parks in whichever op window a +given rank happens to block in — stable per rank (so one rank's per-op numbers look clean and +convincing), arbitrary across ranks, with rank 3's dispatch long exactly where rank 5's combine is, +and bistable across runs of an identical configuration, while the sum, the pair period, is +conserved. A chained per-op median measures where the wait sat on that run rather than what the +operation cost. The cross-rank MIN is the one reduction that removes it, which is why the floors are +published and the medians are not; a p99 of a chained window would mix the same noise back in +through the tail. + +The fresh-entry family keeps its meaning exactly: `components.roundtrip`, `dispatch`, `combine`, +`stage`, `isolated_sum`, `cross_rank_min_us` and `cross_rank_spread_us` are measured and reduced +as they always were, at the same 256×8 sampling, and no stored row was re-meant or re-measured. A +consumer keys on the presence of `components.pair_period`; a row without it predates the chain. +Never rank a chained cell against a pre-chain one on the headline column — `summarize.py` footnotes +its table whenever both appear in it. + +**The published period always means one thing: free-running.** The chain lets ranks drift by up to +about one iteration, which is only sound where the receive plane tolerates it: every backend +double-buffers per dispatch, enforces strict pairing by contract, or completes each op on a reusable +handle, and DeepEP V2's **normal** mode — the one genuinely unaudited cell — was hand-probed with +256 un-synchronized pairs at T=128, EP8+EP16, both precisions (2026-08-06, pin `01dc3aaa`, the +then-current dgxc pool's hybrid GIN over RoCE): all passed, outputs finite, timed inputs unchanged, +cross-rank period agreement within 1µs, and the chain-vs-synced gap is the size of the effect this +section is about (EP8 105.4 vs 125.4µs BF16, 216.6 vs 272.3µs FP8; EP16 838 vs 863µs and 820 vs +897µs). A backend that cannot run free belongs behind a fix, not a measurement variant: re-aligning +ranks between pairs adds its own ~10µs and removes the cross-pair overlap the measurement exists to +capture — a differently-defined quantity that must never share a column with the free-running +period. + +One backend's timed window omits a cost the others pay, deliberately. nccl-ep binds routing with +`ncclEpUpdateHandle`, a collective whose cost scales with the group's token capacity rather than the +token count, so charging it per iteration would import a ladder-max-proportional term into dispatch +-- the same artifact that sizing HT's combine input to the ladder maximum used to put under combine. +It is bound during the untimed warm-up, as NVIDIA's own `ep_bench` does (CUDA events around dispatch +and combine only, handle update outside the loop). Low-latency mode has nothing to exclude: +`ncclEpUpdateHandle` returns immediately and the kernel reads the cached routing inside the timed +dispatch. Every other backend's layout cost scales with tokens and belongs in the window -- uccl-ep +calls `get_dispatch_layout` inside dispatch; deepep-v2, MoRI and FlashInfer pass routing on every +call. + +The artifact records the mode so a reader can keep distinct measurement contracts separate. Every measured component uses one fixed timing profile, defined once in `configs/sweep.json` and baked into every scheduled case: -- 256 trials x 8 timed iterations = 2048 observations; +- 256 trials x 8 timed iterations = 2048 observations for the fresh-entry family; +- 4 chain trials x (128 free-running pairs - 16 dropped for pipeline fill) = 448 observations for + the pair period and another 448 for the op floors, each trial running the two sibling chains + (floors first, then the lean period chain); the trial count is lower than the fresh-entry family's + because one call already yields 128 pairs, and matching it would multiply the leg's wall clock + without buying convergence; - 32 synchronized full dispatch-stage-combine warmups before each available measured component at - every trial/point; + every trial/point, and before each chain trial; - component measurement order rotates each trial (`trial_order`) so every timed component occupies - every position in the sequence, over a per-trial-rotated token ladder; and -- per-iteration maximum latency across ranks before nearest-rank p50/p90/p95/p99. + every position in the sequence, over a per-trial-rotated token ladder, which the chain trials + rotate the same way; and +- per-iteration maximum latency across ranks before nearest-rank p50/p90/p95/p99 (the chained + family reduces by median and minimum instead — see Chained Pair Period). + +`measurement.sampling` carries both halves of that profile, because `sample_count` alone cannot be +decomposed back into them and a 128x4 chain is not the same measurement as a 512x1 one. -Measured roundtrip p99 is the headline latency. Decode and prefill identify the serving regime +The chained pair period is the headline latency where a row carries one, and measured roundtrip +p99 otherwise. Decode and prefill identify the serving regime represented by one MoE-layer collective; they do not change the timed primitive at an otherwise identical shape. Ascending through the ladder, each measured shape is conditioned with 8 untimed full roundtrips — settling clocks, fabric, and buffer state — before it is correctness-checked; @@ -156,12 +434,16 @@ Logical payload bandwidth is: Payload bytes use rank-deduplicated token-rank activations and exclude expert metadata, padding, and backend buffer capacity. BF16 moves 2 bytes per value with no scale payload; an FP8 -dispatch moves 1 byte per value, plus per-128-block FP32 scales for DeepEP's and UCCL-EP's blockwise -codec (none for MoRI's plain e4m3 cast), while combine stays BF16 — so the dispatch and combine directions can carry +dispatch moves 1 byte per value, plus per-128-block FP32 scales for every blockwise codec here — +DeepEP V2, UCCL-EP and FlashInfer EP, which carries them as a fourth dispatch payload — and none for +MoRI's plain e4m3 cast, while combine stays BF16 — so the dispatch and combine directions can carry different byte counts and the roundtrip is their per-field sum. The rank-deduplicated count is exact -for the normal-mode layout; the low-latency layout sends one copy per (token, expert) assignment -rather than per (token, rank), so for a token whose experts share a destination rank this logical -count is a lower bound on the bytes the low-latency kernels actually move. Latency (the headline) is +for the normal-mode layout, and for a low-latency kernel that deduplicates per rank (MoRI's +`IntraNodeLL`, whose combine is an unweighted rank-sum). The low-latency kernels that apply top-k +weights inside combine instead send one copy per (token, expert) assignment rather than per +(token, rank), so for a token whose experts share a destination rank this logical count is a lower +bound on the bytes those kernels move. Each row states which basis it used in `logical_copies`, so +the two are never silently mixed. Latency (the headline) is measured directly and is unaffected. Algorithm bandwidth, bus bandwidth, wire utilization, and physical-link utilization are not emitted without a defined primitive model or transport counters. Logical bandwidth must never be labeled physical bandwidth. Payload and token @@ -177,8 +459,88 @@ routing cannot pass an identity roundtrip. For every rank and point it verifies: 1. destination rank/expert, source token, multiplicity, gate weight, and receive counts; 2. dispatched payload and metadata before timing; 3. combined output before timing; -4. unchanged semantic inputs through all timed samples; and -5. dispatched payload/metadata and combined output again after timing. +4. unchanged semantic inputs through all timed samples; +5. dispatched payload/metadata and combined output again after timing; +6. the **free-running chain's own final combined output**, once per chain trial, against a + drained pair through the identical dispatch→combine path; and +7. the same full check as 2-5 once more against the state the free-running chain leaves behind. + +Checks 6 and 7 exist because checks 2-5 only ever see drained calls: without them the headline +`pair_period` would come from a regime no oracle had inspected, and a backend that transports +correctly when drained but corrupts under back-to-back pairs would present as the fastest cell in +the suite. They answer different questions. Check 6 is a regime A/B, not an oracle: after each +chain trial's closing synchronize — outside every timed window — the chain's last combined output +is compared elementwise (the oracle's tolerance, not bit equality, since combine kernels are not +required to be order-deterministic) against a freshly drained pair through the same code path, so +it catches corruption the chain wrote into its own results. Check 7 reruns the full expert oracle +after the final trial, on the settled communicator (`benchmark_chain` ends synchronized), so it +catches state the chain corrupted for whatever runs next. Interior chain pairs stay unvalidated by +design: each pair overwrites its predecessor's output, and holding or reducing every output would +put device work (or ~O(iters × T × hidden) memory) inside the timed loops the chain exists to keep +clean — a defect confined to interior pairs that heals by the final pair is outside this suite's +evidence. Check 6 is reported per row as `correctness.chain_last_output_passed` (ANDed across +trials) and check 7 as `correctness.post_chain_state_passed`. Check 7 is **folded into +`correctness.passed`**, so its failure fails the leg exactly as any other oracle failure does. + +Check 6 gates **only where the chain stages per pair**, and is reported as `null` elsewhere. That +boundary is measured, not assumed. Wherever `stage_excluded_from_roundtrip` holds — every FP8 +adapter by default, since `stage_device_work` *is* the FP8 flag — the chain hoists staging out of +the timed loop, capturing one warm-up dispatch's stand-in and reusing it for all 128 pairs. Neither +the chain's final combine nor the drained reference then consumes an input matching its own +dispatch, so they are two differently mismatched pairs and nothing requires them to agree. + +The A/B, on h100/deepep-v2/EP8, identical in every respect but the hoist: + +| staging | `chain_last_output_error` | vs `COMBINE_REL_TOL` | +|---|--:|--:| +| hoisted (`fp8_consume=native`) | 31 – 93 | **1000× – 2966×** | +| per pair (`fp8_consume=dequant`) | `0.0` at every rung | bit-identical | + +in **both** `normal` and `low-latency` mode (runs 31180411148, 31185184372, 31185233991), against a +BF16 control that is `0.0` because BF16 never hoists. So the difference was an artifact of the +hoist, not a transport defect: gating on it under the hoist reddened every FP8 leg fleet-wide for +something the harness does deliberately. + +Passing the chain's staged input to the drained pair was tried first and is *not* sufficient — it +makes the two share an input, but a shared input that matches neither dispatch. Only per-pair +staging makes the regimes comparable. + +The check keeps its teeth exactly where it has meaning: every BF16 row, and any FP8 row run under +the `dequant` hatch. `null` there means the question was not asked, never a comparison that ran and +failed. If this boundary is ever moved, move it on a measured magnitude: the verdict alone +justified two wrong calls in one day, and the number settled it in one probe each time. + +What the null leaves uncovered is the intersection of three conditions: a corruption that manifests +only under free-running pairs (the drained oracles are blind by regime), leaves no state behind +(the post-chain oracle is blind), and lives in a path the BF16 sibling row does not exercise (its +still-gating check is blind). Anything short of all three still reds a gate — the DeepEP +low-latency 256-rung corruption hit both precisions, so its BF16 rows would red +`chain_last_output_passed` today, and a wedge trips the per-case hang guard. Note the hoisted chain +also never consumes its own FP8 receive, so the exposure covers dispatch-side corruption as well as +combine-side, and `chain_health` is a consistency guard, not a work guard: a regime defect that +uniformly shortens the combine would publish a fast period no surviving check contradicts. The +standing probe for the intersection is the `dequant` hatch — `CX_FP8_CONSUME=dequant` stages every +pair from its own dispatch, restoring this check on FP8's own free-running pairs. Run one whenever +an FP8 chained period moves in a way its BF16 sibling does not, and before first publishing a new +FP8 backend. flashinfer-ep hoists at BF16 as well (its stage is the workspace staging copy), so its +rows are null at every precision and the hatch does not reach it; its coverage is an open +follow-up. + +Check 6 also publishes `correctness.chain_last_output_error` — the worst chained-vs-drained +relative error, cross-rank MAX, **reported whether or not the verdict passed**. Read it before +reading the verdict. The check assumes a regime defect lands orders of magnitude past +`COMBINE_REL_TOL` while ordinary kernel non-determinism stays well inside it, and that assumption +is not free: FlashInfer EP accumulates its combine in the payload dtype (a BF16 slot-tree) rather +than FP32, so its rounding over a large receive plane is far coarser than a backend reducing in +FP32. A verdict alone cannot separate "the transport corrupted this" from "this tolerance is too +tight for this accumulator", and the two call for opposite responses. The magnitude is the +discriminator; a failure reported without it should not be read as evidence of corruption. +(`post_chain_state_passed` was previously published as `chain_regime_passed`, a name that +overclaimed — a fresh post-chain oracle proves the state, not the chain's outputs. Artifacts from +older harnesses carry the old name, where `null` meant the chain never ran; a validated chain +budget is now required up front, so `post_chain_state_passed` is always a boolean in new +artifacts. `chain_last_output_passed` is the separate case described above — legitimately `null` +wherever the chain hoists staging, which is every FP8 row outside the `dequant` hatch.) Normal-mode adapters use activation-only, unweighted rank-sum combine. The oracle builds each rank's gate-weighted expert aggregate before combine and derives the expected combine from the values @@ -208,24 +570,39 @@ as a correctness property. ## Result Artifact -One raw case document carries `record_type: "case-attempt"` and the single `version`, and contains: +One raw case document carries `record_type: "case-attempt"`, the single `version`, and a +`generated_at` timestamp, and contains: - `identity`: `case_id`, `attempt_ordinal`, `case_factors` (SKU and the scheduled case — backend, EP size, mode, precision, phase, suite, workload, and the topology coordinate), and `allocation_factors` (run id, run attempt, source SHA); - `workload`: `cross_rank_consistent`, whether the routing trace was proven identical across ranks; - `measurement`: dispatch/combine dtype (the realized wire formats — combine always BF16, dispatch - BF16 or the SKU's FP8 format) and semantics, `sampling`, and the per-point `rows`; + BF16 or the SKU's FP8 format) and semantics, `payload_unit` (`token-rank`), `sampling`, and the + per-point `rows`; - `implementation`: backend name, kernel generation, and `maturity` — whether a production inference engine can select this transport today (`production` = exposed by vLLM's `--all2all-backend` or SGLang's `--moe-a2a-backend`; `candidate` = a real transport we benchmark that no engine ships a selector for, so its numbers describe the library rather - than a deployable configuration). The same map is in the registry's `backend_maturity`; -- `topology`: requested SKU/product, placement, nodes, scale-up domain, transport, and world size; + than a deployable configuration). The same map is in the registry's `backend_maturity`. It also + carries `fp8_consume` (which FP8 consumption path the chained roundtrip modelled — see above), + `combine_reduction` and `library_version` (which reduction the oracle held the kernel to, and + the installed library that selected it), and two generation discriminators: + `stage_excluded_from_roundtrip` (whether `roundtrip` excludes expert-output staging, discussed + above) and `chained_period` (whether this document's rows carry the chained family at all); +- `topology`: requested SKU/product, placement, `gpus_per_node`, nodes, scale-up domain, `scope`, + `topology_class`, world size, and three distinct transport fields — `scale_up_transport` and + `scale_out_transport` (the components), plus `transport`, the derived summary that folds them + into one string (`nvlink` scale-up-only, `nvlink-rdma` once a case scales out); +- `runtime`: the realized software stack — `vendor`, `framework` (the torch version), + `accelerator_runtime` (the CUDA or HIP version torch was built against), and + `collective_library` (`nccl`/`rccl` and the version actually loaded into the process); - `provenance`: the mounted image tag and source SHA; and - `outcome`: `status` (`success` or `invalid`) and `reasons`. -Each `rows` entry carries point latency, byte accounting, token rate, correctness, load, and fanout; +Each `rows` entry carries point latency (the fresh-entry `components` plus the chained +`components.pair_period`, `chain_floor_us` and `chain_health` — see Chained Pair Period), byte +accounting, token rate, correctness, load, and fanout; per-point statistics are summarized in place, not emitted as separate documents. Each dispatched case writes exactly this one raw result document; unsupported or never-run cells produce no synthetic record. diff --git a/experimental/CollectiveX/launchers/launch_gb-nv.sh b/experimental/CollectiveX/launchers/launch_gb-nv.sh index 2a4f258e9..855c16fd6 100644 --- a/experimental/CollectiveX/launchers/launch_gb-nv.sh +++ b/experimental/CollectiveX/launchers/launch_gb-nv.sh @@ -67,9 +67,13 @@ export COLLX_BACKEND_CACHE_ROOT=/cx-cache # ---- scheduler-allocation: salloc the trays --------------------------------- command -v salloc >/dev/null || collx_die "salloc not found" -collx_salloc_jobid --partition="$PARTITION" --account="$ACCOUNT" --nodes="$NODES" \ - --gres=gpu:"$GPN" --ntasks-per-node="$GPN" --exclusive --mem=0 --cpus-per-task=35 \ - --time="$TIME_MIN" +allocation=(--partition="$PARTITION" --account="$ACCOUNT" --nodes="$NODES" + --gres=gpu:"$GPN" --ntasks-per-node="$GPN" --exclusive --mem=0 --cpus-per-task=35 + --time="$TIME_MIN") +# Honour the registry's node denylist. Without this the key is accepted by +# config.py and silently dropped here, so a quarantined tray keeps getting picked. +[ -z "${COLLX_EXCLUDE_NODES:-}" ] || allocation+=(--exclude="$COLLX_EXCLUDE_NODES") +collx_salloc_jobid "${allocation[@]}" [ -n "$JOB_ID" ] || collx_die "no JOB_ID from salloc" # ---- container-import: squash file resolved on the allocation --------------- diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh index eee35d1fa..1f891295f 100644 --- a/experimental/CollectiveX/launchers/launch_mi-amds.sh +++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh @@ -51,7 +51,6 @@ if [ "$NODES" -gt 1 ]; then else export COLLX_TRANSPORT=xgmi fi -export COLLX_RUN_TIMEOUT="${COLLX_RUN_TIMEOUT:-1800}" collx_apply_network_profile "$NODES" "$COLLX_TRANSPORT" collx_require_vars COLLX_IMAGE COLLX_IMAGE_PLATFORM COLLX_PARTITION COLLX_SQUASH_DIR COLLX_STAGE_DIR PARTITION="$COLLX_PARTITION"; SQUASH_DIR="$COLLX_SQUASH_DIR" diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh index f30eef42e..5be8ceda9 100644 --- a/experimental/CollectiveX/launchers/launch_single-slurm.sh +++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh @@ -22,8 +22,10 @@ case "$RUNNER" in PRODUCT=h200; DEFAULT_TIME=45; REQUIRE_ACCOUNT=0 SRUN_EXTRA=(--container-remap-root) ;; - b200-dgxc) - PRODUCT=b200; DEFAULT_TIME=30; REQUIRE_ACCOUNT=1 + b200-nscale) + # Bare-metal B200 (nsc): native IB rails + gdrdrv make the deepep low-latency EP16 rows + # dispatchable, unlike the virtualized dgxc pool. 45 min covers first-run backend builds. + PRODUCT=b200; DEFAULT_TIME=45; REQUIRE_ACCOUNT=1 ALLOC_EXTRA=(--mem=0) ;; b300) @@ -114,17 +116,25 @@ for allocation_attempt in 1 2 3; do elif [ "$RUNNER" = b300 ] \ && ! collx_validate_cuda_context_on_job "$JOB_ID" "$NODES" "$GPN"; then validation_failure=cuda-context + elif ! collx_validate_gpu_health_on_job "$JOB_ID" "$NODES" "$GPN"; then + validation_failure=gpu-health else break fi retryable=0 [ "$RUNNER:$validation_failure" != h100-dgxc:network ] || retryable=1 [ "$RUNNER:$validation_failure" != b300:cuda-context ] || retryable=1 + # A throttled GPU paces every rank, so retrying on another node is right on every SKU. + [ "$validation_failure" != gpu-health ] || retryable=1 if [ "$retryable" = 0 ] || [ "$allocation_attempt" = 3 ]; then if [ "$validation_failure" = network ]; then collx_log_tail "${COLLX_NETWORK_PROFILE_LOG:-}" collx_die "allocated nodes failed the network profile" fi + if [ "$validation_failure" = gpu-health ]; then + collx_log_tail "${COLLX_GPU_HEALTH_LOG:-}" + collx_die "allocated nodes hold a thermally throttled GPU" + fi collx_log_tail "$COLLX_CUDA_CONTEXT_LOG" collx_die "allocated nodes failed accelerator context validation" fi diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh index 24072604a..d786799c7 100644 --- a/experimental/CollectiveX/runtime/common.sh +++ b/experimental/CollectiveX/runtime/common.sh @@ -12,7 +12,14 @@ collx_log() { printf '[collectivex] %s\n' "$*" >&2; } collx_die() { printf '[collectivex] FATAL: %s\n' "$*" >&2; exit 1; } COLLX_DEEPEP_V2_REPO="https://github.com/deepseek-ai/DeepEP" -COLLX_DEEPEP_V2_COMMIT="fa8a9b16898204afd347c663b89e65ef87dc6ce6" +# Upstream main, replacing the pre-merge head of PR #605 (fa8a9b16). #605 merged 2026-04-29 and +# main carries its one unique commit, the #630 single-node V2 init fix, as 56169594e -- plus fixes +# the branch never received: #642 fence.proxy.async.shared::cta in LOW_LATENCY_COMBINE_RECV, which +# fixes the Blackwell low-latency combine corruption at the top ladder rung (DeepEP issue #700); +# #715 system-scope release before the GIN barrier when scale-up spans NVLink and RDMA; #688 NCCL +# Device API compat; #178 SM90; #641 internode dispatch args; #640/#627 NVSHMEM/NCCL SO-name +# resolution for pip wheels. The backend cache is keyed on this value, so a change forces a rebuild. +COLLX_DEEPEP_V2_COMMIT="01dc3aaac82068020353dce2c302e38153c0bfaa" COLLX_UCCL_REPO="https://github.com/uccl-project/uccl" COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3" @@ -514,8 +521,6 @@ collx_prepare_deepep_source() { && [ "$(git -C "$temporary" rev-parse HEAD)" = "$COLLX_DEEPEP_V2_COMMIT" ] \ && GIT_TERMINAL_PROMPT=0 git -C "$temporary" submodule update -q --init --depth 1 \ third-party/fmt >> "$log" 2>&1 \ - && python3 "$COLLX_RUNTIME_DIR/stage.py" rewrite-deepep-v2 \ - "$temporary/deep_ep/__init__.py" >> "$log" 2>&1 \ && mv -- "$temporary" "$source" >> "$log" 2>&1; then return 0 fi @@ -603,10 +608,16 @@ collx_prepare_stage_dir() { "${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-}}")" \ || collx_die "canonical CollectiveX execution cannot create an isolated stage directory" ;; - h200-dgxc|b200-dgxc) + h200-dgxc) COLLX_STAGE_DIR="$(collx_prepare_implicit_stage_base)" \ || collx_die "canonical CollectiveX execution cannot create an isolated stage directory" ;; + b200-nscale) + # Anchor at the squash dir's parent (/data/home/sa-shared): the passwd home is not + # compute-visible. + COLLX_STAGE_DIR="$(collx_prepare_implicit_stage_base "${COLLX_SQUASH_DIR%/*}")" \ + || collx_die "canonical CollectiveX execution cannot create an isolated stage directory" + ;; mi300x|mi325x|mi355x) COLLX_STAGE_DIR="$(collx_prepare_runner_shared_stage_base)" \ || collx_die "canonical AMD execution cannot create an isolated shared stage directory" @@ -706,7 +717,18 @@ collx_ensure_squash() { # architecture. The squash directory must be shared with the submit host. collx_ensure_squash_on_job() { local job_id="$1" squash_dir="$2" image="$3" lock_dir="${4:-}" sq key lock - local log_label=container-import log + local log_label=container-import log attempt rc + # The import writes tens of GB to whatever the operator gave as squash storage, and on some + # clusters that storage is a SOFT-mounted network filesystem, i.e. one that returns an error + # rather than blocking when its transport is briefly unavailable. gb300's /data is NFSv3 over + # RDMA (proto=rdma, soft), and a transport gap there surfaces as `mkdir: cannot create + # directory '/data': Protocol family not supported` -- an address-family errno from mkdir, + # which reads like a missing mount but is not one: the same node mounts and writes it fine + # minutes later. Run 31089556516 lost its gb300 shards to that, ~25 minutes into each leg. + # So a failed import is retried rather than being terminal. Retrying is safe because the + # remote block re-takes the lock and re-checks the squash each time, removing a partial file + # before re-importing. + local max_attempts="${COLLX_IMPORT_ATTEMPTS:-3}" [[ "$job_id" =~ ^[0-9]+$ ]] || return 1 case "${COLLX_SALLOC_ATTEMPT:-1}" in 1) ;; @@ -718,13 +740,21 @@ collx_ensure_squash_on_job() { key="${key%.sqsh}" [ -n "$lock_dir" ] || lock_dir="$squash_dir/.locks" lock="$lock_dir/${key}.lock" - log="$(collx_private_log_path "$log_label")" - # Run once per node because some clusters use node-local squash storage. - if ! srun --jobid="$job_id" --nodes="${COLLX_NODES:-1}" --ntasks="${COLLX_NODES:-1}" \ + for attempt in $(seq 1 "$max_attempts"); do + # A per-attempt log: collx_private_log_path truncates, so reusing one path would erase the + # evidence of the failure that caused the retry. + if [ "$attempt" -eq 1 ]; then + log="$(collx_private_log_path "$log_label")" + else + log="$(collx_private_log_path "${log_label}-r${attempt}")" + fi + rc=0 + # Run once per node because some clusters use node-local squash storage. + srun --jobid="$job_id" --nodes="${COLLX_NODES:-1}" --ntasks="${COLLX_NODES:-1}" \ --ntasks-per-node=1 --chdir=/tmp \ --export="$(collx_host_exports)" \ bash -s -- "$sq" "$lock" "$image" "$COLLX_IMAGE_PLATFORM" \ - > "$log" 2>&1 <<'BASH' + > "$log" 2>&1 <<'BASH' || rc=$? set -euo pipefail sq="$1"; lock="$2"; image="$3"; platform="$4" machine="$(uname -m)" @@ -752,12 +782,43 @@ else unsquashfs -l "$sq" >/dev/null 2>&1 fi BASH - then - collx_log "ERROR: container import failed" - collx_log_tail "$log" - return 1 - fi - printf '%s' "$sq" + [ "$rc" = 0 ] && { printf '%s' "$sq"; return 0; } + # 13 is the remote block's architecture guard: the image platform does not match the + # allocated machine. That is a property of the case, not of the moment, so it never + # improves on a retry and burning two more attempts on it only delays the real message. + if [ "$rc" = 13 ]; then + collx_log "ERROR: container image platform does not match the allocated architecture" + collx_log_tail "$log" + return 1 + fi + if [ "$attempt" -lt "$max_attempts" ]; then + collx_log "container import attempt $attempt/$max_attempts failed (rc=$rc); retrying" + collx_log_tail "$log" + sleep "$((attempt * 30))" + fi + done + collx_log "ERROR: container import failed after $max_attempts attempts" + collx_log_tail "$log" + return 1 +} + +# Reject an allocation whose GPUs are throttled: collectives are barriers, so one clamped device +# paces every rank. `--gres` mirrors the cuda-context probe below so the step provably sees the +# devices it judges; `--time` bounds nvidia-smi wedging in D-state on the sick hardware itself, +# which Python's own timeout cannot reap. +collx_validate_gpu_health_on_job() { + local job_id="$1" nodes="$2" gpus_per_node="$3" log_label=gpu-health log + case "${COLLX_SALLOC_ATTEMPT:-1}" in + 1) ;; + 2|3) log_label+="-a${COLLX_SALLOC_ATTEMPT}" ;; + *) return 1 ;; + esac + log="$(collx_private_log_path "$log_label")" + export COLLX_GPU_HEALTH_LOG="$log" + srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \ + --gres=gpu:"$gpus_per_node" --time=5 --chdir=/tmp --input=all \ + --export="$(collx_host_exports)" python3 /dev/stdin gpu-health \ + < "$COLLX_RUNTIME_DIR/probe.py" >"$log" 2>&1 } # A clean nvidia-smi inventory does not prove that a prior cancelled workload @@ -906,7 +967,14 @@ collx_run_shard() { || { rm -f "$argv_file"; collx_die "case $ci produced no benchmark arguments"; } collx_log "EP${NGPUS}[$((ci + 1))/$expected_cases] $COLLX_BENCH" runtime_log="$(collx_private_log_path "runtime-c$(printf '%03d' "$ci")")" - if ! timeout -k 30 "${COLLX_RUN_TIMEOUT:-900}" \ + # A hang guard, not a work budget: at 900 it killed FP8 prefill cases that had already written + # complete artifacts, and at 1800 it killed b200 and h200 multi-node EP16 prefill (run + # 31020463440). Those two slowed because both pools were virtualized and their GPU-NIC p2p is + # degraded -- h200 sustains ~34 GB/s per node against a nominal 8x400G where bare-metal h100 + # reaches wire rate on the same 2-node RDMA+GIN topology, which is why h100 never moved. See + # docs/methodology.md. Truncating a real measurement is worse than a late one, and 5400 stays + # inside the 300-minute allocation. + if ! timeout -k 30 "${COLLX_RUN_TIMEOUT:-5400}" \ srun --jobid="$JOB_ID" --nodes="$NODES" \ --ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \ --container-name="$container_name" --container-image="$SQUASH_FILE" \ diff --git a/experimental/CollectiveX/runtime/config.py b/experimental/CollectiveX/runtime/config.py index d1e6132bd..ae928550f 100644 --- a/experimental/CollectiveX/runtime/config.py +++ b/experimental/CollectiveX/runtime/config.py @@ -17,6 +17,42 @@ "socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level", "rdma_traffic_class", "rail_isolated", } +# Timing knobs, in the order the legacy colon-string encoded them, paired with the run_ep flag +# each one drives. The names match configs/sweep.json so a case's timing block is readable +# rather than positional. +_TIMING_FLAGS = ( + ("iters_per_trial", "--iters"), + ("trials_per_point", "--trials"), + ("warmup_iters_per_trial", "--warmup"), + ("chain_iters_per_trial", "--chain-iters"), + ("chain_trials_per_point", "--chain-trials"), + ("chain_drop", "--chain-drop"), +) + + +def _migrate_timing(timing: object) -> dict: + """The one place a legacy colon-string timing profile is decoded. + + Current shards carry an object keyed by the _TIMING_FLAGS names. A replayed pre-chain + shard carries "iters:trials:warmup" and a replayed post-chain one all six, positionally; + three fields emit no --chain-* flags, so run_ep's argparse defaults supply them rather + than this file duplicating the values. Anything else fails closed. + + Worth knowing when replaying an old shard: timing was never part of case_id, so a + pre-chain case re-run today lands under its original identity while measured with + today's chain budget. + """ + names = tuple(key for key, _ in _TIMING_FLAGS) + if isinstance(timing, dict): + if set(timing) not in (set(names), set(names[:3])): + print(f"unrecognised timing object {timing!r}", file=sys.stderr) + raise SystemExit(1) + return timing + fields = str(timing).split(":") + if len(fields) not in (3, 6): + print(f"unrecognised timing profile {timing!r}", file=sys.stderr) + raise SystemExit(1) + return dict(zip(names, fields)) def _platforms() -> dict: @@ -125,16 +161,14 @@ def _emit_argv(case: dict, version: object, runner: str, ts: str, index: int) -> "--workload-name", str(case["workload"]), "--version", str(version), ] - iters, trials, warmup = str(case["timing"]).split(":") - for flag, value in (("--iters", iters), ("--trials", trials), ("--warmup", warmup)): - argv += [flag, value] - # precision is part of the filename so a cell's bf16 and fp8 legs (distinct shards - # sharing runner/backend/phase and each numbering cases from index 0) cannot collide - # when they land in the shared results/ dir under the same second-resolution ts. - out = ( - f"results/{runner}_{case['backend']}_{case['precision']}_{case['phase']}" - f"_{ts}-c{index:03d}.json" - ) + timing = _migrate_timing(case["timing"]) + for key, flag in _TIMING_FLAGS: + if key in timing: + argv += [flag, str(timing[key])] + # case_id is the canonical identity (sku==runner, backend, workload, mode, phase, ep, routing, + # precision), so a new identity axis cannot be omitted from the filename the way mode once was. + # ts + the per-shard case index disambiguate legs that share one results/ directory. + out = f"results/{case['case_id']}_{ts}-c{index:03d}.json" argv += ["--out", out] sys.stdout.buffer.write(b"\0".join(part.encode() for part in argv) + b"\0") diff --git a/experimental/CollectiveX/runtime/probe.py b/experimental/CollectiveX/runtime/probe.py index 922a99682..5f4fe024e 100644 --- a/experimental/CollectiveX/runtime/probe.py +++ b/experimental/CollectiveX/runtime/probe.py @@ -19,7 +19,9 @@ def default_route_interface(route_path: Path = Path("/proc/net/route")) -> str: def prepare_cache(parent_path: str) -> str: path = Path(parent_path).resolve() / f".collectivex-backend-cache-{os.getuid()}" - path.mkdir(mode=0o700, exist_ok=True) + # parents=True: this runs before the first container import, so a fresh pool's squash_dir may + # not exist yet (b200-nscale run 31092445934). 0o700 applies to the cache dir, not its parents. + path.mkdir(mode=0o700, parents=True, exist_ok=True) os.chmod(path, 0o700) return str(path) @@ -31,6 +33,96 @@ def validate_cuda_context(expected: int) -> None: raise SystemExit(1) +_GPU_HEALTH_FIELDS = ("index", "clocks_event_reasons.sw_thermal_slowdown", + "clocks_event_reasons.hw_thermal_slowdown", "temperature.gpu") + + +def gpu_health_faults(output: str, max_temperature_c: int = 90) -> list[str]: + """Throttled or overheating GPUs in an `nvidia-smi --format=csv,noheader` block. + + Split out from the I/O so parsing is testable without hardware; see + tests/test_runtime.py::GpuHealthProbe. Returns [] for anything unreadable -- the caller treats + an unreadable probe as healthy rather than blocking a leg on it. + """ + faults = [] + for line in output.splitlines(): + cells = [cell.strip() for cell in line.split(",")] + if len(cells) != len(_GPU_HEALTH_FIELDS): + continue + index, software, hardware, temperature = cells + # "Not Active" is the healthy reading, so compare exactly -- a substring test for + # "Active" passes the fault straight through. + throttled = "Active" in (software, hardware) + try: + too_hot = int(temperature.split()[0]) > max_temperature_c + except (IndexError, ValueError): + too_hot = False + if throttled or too_hot: + faults.append( + f"gpu {index}: sw_thermal={software} hw_thermal={hardware} temp={temperature}" + ) + return faults + + +def gpu_temperature_spread(output: str) -> tuple[int, int, int] | None: + """`(hottest, median, spread)` GPU temperature, or None if unreadable. + + Reported, not gated on: the absolute threshold in `gpu_health_faults` can be unreachable (an + H100 clamps at ~86-87 C, under the 90 C gate), and in the one measured fault the only + pre-flight signal was relative -- the sick GPU idled at 55 C against ~30 C for its siblings. + Healthy references: 50-66 C under load on h100, 34-39 C on b200. + """ + temperatures = [] + for line in output.splitlines(): + cells = [cell.strip() for cell in line.split(",")] + if len(cells) != len(_GPU_HEALTH_FIELDS): + continue + try: + temperatures.append(int(cells[3].split()[0])) + except (IndexError, ValueError): + continue + if not temperatures: + return None + temperatures.sort() + median = temperatures[len(temperatures) // 2] + return temperatures[-1], median, temperatures[-1] - median + + +def validate_gpu_health(max_temperature_c: int = 90) -> None: + """Reject an allocation holding a thermally throttled GPU. + + Every collective is a barrier, so one clamped device paces every rank: a B200 with GPU 7 at + 120 MHz ran a case 17x slower and was killed twice by the wall-clock guard. Gate on the throttle + flag, not the clock -- an idle B200 also reads 120 MHz -- with temperature as an independent + second signal. Fails open on anything unreadable: no `nvidia-smi`, non-zero exit, bad output. + """ + import shutil + import subprocess + + if shutil.which("nvidia-smi") is None: + return + try: + output = subprocess.run( + ["nvidia-smi", f"--query-gpu={','.join(_GPU_HEALTH_FIELDS)}", + "--format=csv,noheader"], + capture_output=True, text=True, timeout=60, check=True, + ).stdout + except (OSError, subprocess.SubprocessError): + return + faults = gpu_health_faults(output, max_temperature_c) + for fault in faults: + _emit(f"gpu-health-fault {fault}") + if faults: + raise SystemExit(1) + # Positive control: without it a blind gate -- no visible devices, or a driver spelling these + # fields `clocks_throttle_reasons.*` -- is indistinguishable from a healthy pass. + spread = gpu_temperature_spread(output) + detail = "" if spread is None else f" hottest={spread[0]}C median={spread[1]}C spread={spread[2]}C" + _emit( + f"gpu-health-checked gpus={sum(1 for line in output.splitlines() if line.strip())}{detail}" + ) + + def _emit(marker: str) -> None: # collx_validate_network_profile_on_job (runtime/common.sh) greps these exact strings # out of the per-node probe log to derive COLLX_SOCKET_IFNAME / COLLX_RDMA_LINK_LAYER and to @@ -113,11 +205,13 @@ def main() -> None: commands.add_parser("default-route-interface") command = commands.add_parser("prepare-cache"); command.add_argument("parent") command = commands.add_parser("cuda-context"); command.add_argument("expected", type=int) + commands.add_parser("gpu-health") command = commands.add_parser("network-profile"); command.add_argument("socket_names"); command.add_argument("rdma_devices"); command.add_argument("gid_index") args = parser.parse_args() if args.command == "default-route-interface": print(default_route_interface(), end="") elif args.command == "prepare-cache": print(prepare_cache(args.parent), end="") elif args.command == "cuda-context": validate_cuda_context(args.expected) + elif args.command == "gpu-health": validate_gpu_health() else: validate_network_profile(args.socket_names, args.rdma_devices, args.gid_index) diff --git a/experimental/CollectiveX/runtime/stage.py b/experimental/CollectiveX/runtime/stage.py index 119b77496..a3231f80b 100644 --- a/experimental/CollectiveX/runtime/stage.py +++ b/experimental/CollectiveX/runtime/stage.py @@ -60,15 +60,6 @@ def validate_cleanup(args) -> None: raise SystemExit(1) -def rewrite_deepep_v2(args) -> None: - path = Path(args.path) - old = "for so in [line.strip().split(' ')[-1] for line in f if 'nccl' in line]:" - new = "for so in [line.strip().split(' ')[-1] for line in f if 'libnccl' in line]:" - text = path.read_text() - if text.count(old) != 1: raise SystemExit(1) - path.write_text(text.replace(old, new)) - - # The runtime/common.sh launcher shells out to these subcommands by literal name and # positional argv; there are no optional flags. That argv shape is a string contract with # common.sh — a subcommand or flag common.sh passes but this parser does not declare fails @@ -80,7 +71,7 @@ def rewrite_deepep_v2(args) -> None: "resolve-directory": (("path",),), "validate-stage-path": (("repo",), ("base",), ("child",), ("job_root", "?"), ("workspace", "?")), "create-stage": (("stage",),), "copy-repository": (("source",), ("target",)), - "validate-cleanup": (("root",),), "rewrite-deepep-v2": (("path",),), + "validate-cleanup": (("root",),), } diff --git a/experimental/CollectiveX/summarize.py b/experimental/CollectiveX/summarize.py index 5785d6757..47e58826b 100644 --- a/experimental/CollectiveX/summarize.py +++ b/experimental/CollectiveX/summarize.py @@ -15,8 +15,6 @@ def load_results(directory: str, runner: str | None, timestamp: str | None) -> list[dict]: documents: list[dict] = [] for path in sorted(Path(directory).glob("*.json")): - if runner and not path.name.startswith(f"{runner}_"): - continue if timestamp and timestamp not in path.name: continue try: @@ -24,8 +22,15 @@ def load_results(directory: str, runner: str | None, timestamp: str | None) -> l document = json.load(handle) except (OSError, ValueError): continue - if isinstance(document, dict) and document.get("record_type") == CASE_RECORD_TYPE: - documents.append(document) + if not (isinstance(document, dict) and document.get("record_type") == CASE_RECORD_TYPE): + continue + # Filter on the SKU the row declares, not on the filename. Results are named + # `_-cNNN.json` and case_id joins its factors with "-", so the old + # `startswith(f"{runner}_")` test could never match: the summary would have + # rendered an empty table rather than failing. + if runner and document.get("identity", {}).get("case_factors", {}).get("sku") != runner: + continue + documents.append(document) return documents @@ -43,11 +48,65 @@ def _identity(document: dict) -> tuple[str, str, str, str, str, str, int, str]: ) -def _headline(document: dict) -> tuple[int | str, float | str, float | str]: +def _topology(document: dict) -> str: + """Scale-up shape, because the same `ep` label is not the same hardware. + + GB200/GB300 run 4 GPUs per node in a 72-GPU MNNVL domain, so their EP8 spans two trays while + every other SKU's EP8 is one node over NVLink or XGMI. Printing it stops the false comparison. + """ + topology = document.get("topology") or {} + per_node = topology.get("gpus_per_node") + domain = topology.get("scale_up_domain") + nodes = topology.get("nodes") + if per_node is None or domain is None: + return "-" + return f"{nodes}x{per_node}/d{domain}" + + +def _wire_basis(document: dict) -> str: + """Which copy basis this backend's kernels actually move. + + Low-latency deepep-v2/uccl-ep/nccl-ep receive one copy per (token, expert); MoRI's IntraNodeLL + deduplicates by destination rank -- ~1.5x different combine traffic at EP8, so not equal work. + """ + rows = document["measurement"]["rows"] + copies = (rows[0] if rows else {}).get("logical_copies") or {} + return {"per-assignment": "assign", "rank-deduplicated": "dedup"}.get(copies.get("wire"), "-") + + +def _headline(document: dict) -> tuple: + """Headline row, with the skew bracket beside it. + + `p50`/`p99` are the chained pair period where the row carries one — what a decode loop pays + per MoE layer, cross-rank median over back-to-back pairs. Otherwise `roundtrip`, an + idle-pipeline latency reduced by cross-rank MAX. Different quantities, so the last tuple + element reports which the row carries and `render` footnotes the table accordingly. + + That MAX charges entry stagger to the operation, by an amount that is a property of the + backend — on identical h200 low-latency cells the per-iteration spread is 9.2us for + deepep-v2/uccl-ep against 2.0us for nccl-ep. `min50` (MIN over the same iterations) and `skew` + (per-iteration MAX-MIN) bracket it. + """ rows = document["measurement"]["rows"] + if not rows: + # Degrade rather than crash: one malformed shard must not lose the whole table. No row + # means nothing to attribute, so it votes on neither footnote. + return ("-", "-", "-", "-", "-", None) row = next((item for item in rows if item["tokens_per_rank"] == 64), rows[len(rows) // 2]) - latency = row["components"]["roundtrip"]["percentiles_us"] - return row["tokens_per_rank"], latency["p50"], latency["p99"] + period = (row["components"].get("pair_period") or {}).get("percentiles_us") + latency = period or row["components"]["roundtrip"]["percentiles_us"] + + def percentile(block: str, name: str) -> float | str: + # Absent on rows measured before the skew diagnostics were emitted. + component = (row.get(block) or {}).get(name) or row.get(block) or {} + return (component.get("percentiles_us") or {}).get("p50", "-") + + return ( + row["tokens_per_rank"], latency["p50"], latency["p99"], + percentile("cross_rank_min_us", "roundtrip"), + percentile("cross_rank_spread_us", ""), + period is not None, + ) def render(documents: list[dict]) -> str: @@ -63,19 +122,45 @@ def render(documents: list[dict]) -> str: ) lines.append("") lines += [ - "| ver | sku | backend | mode | precision | suite | phase | routing | ep | outcome | T* | p50 us | p99 us |", - "|--:|---|---|---|---|---|---|---|--:|---|--:|--:|--:|", + "| ver | sku | backend | mode | precision | suite | phase | routing | ep | topo " + "| wire | outcome | T* | p50* us | p99* us | min50 us | skew us |", + "|--:|---|---|---|---|---|---|---|--:|---|---|---|--:|--:|--:|--:|--:|", ] + chained = [] for document in documents: sku, backend, suite, routing, mode, phase, ep, precision = _identity(document) - token, p50, p99 = _headline(document) + token, p50, p99, min50, skew, row_chained = _headline(document) + if row_chained is not None: + chained.append(row_chained) + topo, wire = _topology(document), _wire_basis(document) lines.append( f"| {document['version']} | {sku} | `{backend}` | {mode} | {precision} | {suite} | " - f"{phase} | {routing} | {ep} | " - f"{document['outcome']['status']} | {token} | {p50} | {p99} |" + f"{phase} | {routing} | {ep} | {topo} | {wire} | " + f"{document['outcome']['status']} | {token} | {p50} | {p99} | {min50} | {skew} |" ) if not documents: lines.append("\n> No valid native outcome documents found.") + # The starred columns can hold two different quantities, so the table always says which — and + # says so loudly when it holds both, since a mixed column silently compares a steady-state + # period against an idle-pipeline latency. + if chained: + carrying = sum(1 for flag in chained if flag) + fallbacks = chained.count(False) + period_note = ("`*` chained pair period (back-to-back pairs, cross-rank median) — " + "what a decode loop pays per layer") + if fallbacks and carrying: + lines.append( + f"\n> {period_note}; **{fallbacks} of {len(chained)} row(s) predate it** and " + "fall back to the drained `roundtrip` (cross-rank MAX). The two are different " + "quantities — do not rank across them." + ) + elif fallbacks: + lines.append( + "\n> `*` drained `roundtrip` (cross-rank MAX): no row here carries a chained " + "pair period." + ) + else: + lines.append(f"\n> {period_note}.") return "\n".join(lines) diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py index 11bab352c..a99613d68 100644 --- a/experimental/CollectiveX/sweep_matrix.py +++ b/experimental/CollectiveX/sweep_matrix.py @@ -41,11 +41,18 @@ def _load_config(name: str) -> dict[str, Any]: # NCCL EP is BF16-only on the strength of RELEASE.md's "No FP8 support" row, which is # worth re-testing — see the note in bench/ep_nccl.py. "nccl-ep": ("bf16",), - # FlashInfer one-sided is BF16-only this pass: the combine side accepts FP8 output - # dtypes, but an FP8 dispatch needs the scale payload plumbed as a second - # input_payload and validated against the oracle cast round-trip. - "flashinfer-ep": ("bf16",), + # FlashInfer FP8 is dispatch-side only (scales as a fourth payload, combine stays BF16), + # and uses the same per-128-block e4m3 recipe as deepep-v2/uccl-ep so the axis is + # comparable. Realizable, but off every deployed path -- see OFF_PATH_PRECISIONS. + "flashinfer-ep": ("bf16", "fp8"), } +# Precisions a backend REALIZES but that no serving engine can select on that transport. Kept +# out of the default matrix so a production sweep measures deployable configurations, and still +# reachable by naming the precision explicitly (`--precisions fp8`) for transport comparison. +# vLLM accepts only nvfp4/mxfp8/bf16 on FlashInfer's one-sided all-to-all, so its FP8 row +# measures the collective off any path an engine selects. Declared here rather than on the +# adapter because this generator must resolve the matrix with no vendor imports. +OFF_PATH_PRECISIONS = {"flashinfer-ep": ("fp8",)} # Short shard-ID slug per non-normal mode. Normal-mode shard IDs carry no mode # segment so existing references stay valid; a low-latency shard adds "-ll". _MODE_SLUG = {"low-latency": "ll"} @@ -135,11 +142,23 @@ def resolve_matrix( raise SystemExit("--only-sku and --exclude-skus select disjoint pools") timing = SWEEP["timing"] - timing_profile = ":".join(str(timing[key]) for key in ( + # Passed through as an object keyed by the sweep.json names. runtime/config.py maps each + # key to its run_ep flag and holds the legacy colon-string decode in one migration function. + timing_profile = {key: int(timing[key]) for key in ( "iters_per_trial", "trials_per_point", "warmup_iters_per_trial", - )) + "chain_iters_per_trial", "chain_trials_per_point", "chain_drop", + )} workload = SWEEP["workload"] targets = _selected_backends(backend) + # Fail closed on a backend with no declared precisions: defaulting to BF16 would drop its + # fp8 cases from the matrix entirely, and a case that never ran is invisible to every + # downstream gate (run_sweep's non-bf16-dispatch guard only sees cases that did run). + undeclared = [target for target in targets if target not in BACKEND_PRECISIONS] + if undeclared: + raise SystemExit( + f"backends {undeclared} have no BACKEND_PRECISIONS entry; declare their dispatch " + "precisions rather than silently defaulting to bf16" + ) requested_cases: list[dict[str, Any]] = [] shards: dict[tuple[str, str, str, int, str], list[dict[str, Any]]] = {} @@ -157,10 +176,14 @@ def resolve_matrix( if runnable_eps is None: continue runnable = ep in runnable_eps - backend_precisions = BACKEND_PRECISIONS.get(target, ("bf16",)) + backend_precisions = BACKEND_PRECISIONS[target] + # Off-path precisions are dropped unless the caller named the precision + # explicitly, so the default matrix carries only deployable configurations. + off_path = OFF_PATH_PRECISIONS.get(target, ()) supported = [ precision for precision in SWEEP["precisions"] if precision in backend_precisions + and (precision not in off_path or precision in selected_precisions) ] if runnable: # A runnable cell fans out over the modes it realizes at this diff --git a/experimental/CollectiveX/tests/test_backends.py b/experimental/CollectiveX/tests/test_backends.py new file mode 100644 index 000000000..fffc62bd1 --- /dev/null +++ b/experimental/CollectiveX/tests/test_backends.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +"""EPBackend contracts: ladder/spec construction, the staging-vs-roundtrip gate, and the NCCL EP handle.""" +from __future__ import annotations + +import os +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT), str(ROOT / "bench")] + +import ep_backend # noqa: E402 +from ep_backend import EPBackend, RankInputs # noqa: E402 + + +# ---- from test_ep_backend.py ------------------------------------------------------ +def args(**updates): + values = dict( + experts=8, phase="decode", tokens_ladder="", routing="uniform", seed=0, + hidden=16, topk=2, mode="normal", precision="bf16", + ) + values.update(updates) + return types.SimpleNamespace(**values) + + +class FakeBackend(EPBackend): + name = "fake" + + def __init__(self, options, *, cap=None, world_size=1): + super().__init__(options, 0, world_size, 0, "cpu") + self.cap = cap + self.calls: list[str] = [] + + def create_buffer(self, spec): + return None + + def dispatch(self, problem): + self.calls.append("dispatch") + return object() + + def stage(self, problem, handle): + self.calls.append("stage") + + def combine(self, problem, handle): + self.calls.append("combine") + + def recv_tokens(self, handle): + return 0 + + def inspect_dispatch(self, problem, handle): + return None + + def combine_transformed(self, problem, handle, transformed): + return None + + def buffer_cap(self, options): + return self.cap + + def _build_rank_inputs(self, options, tokens): + return RankInputs( + tokens_per_rank=tokens, topk_idx=None, topk_weights=None, + activations=None, + ) + + +class BackendTests(unittest.TestCase): + def test_invalid_or_fully_clamped_ladder_fails_before_execution(self): + for backend, message in ( + (FakeBackend(args(tokens_ladder="0")), "empty token ladder"), + (FakeBackend(args(tokens_ladder="128"), cap=64), "cap=64"), + ): + with self.subTest(message=message): + spec = backend.make_inputs(backend.args) + self.assertEqual(spec.rc, 2) + self.assertIn(message, spec.message) + + def test_mode_is_fail_closed(self): + with self.assertRaises(ValueError): + FakeBackend(args(mode="unsupported")) + + def test_precision_is_fail_closed(self): + # The base SUPPORTED_PRECISIONS is BF16-only; an adapter that has not opted + # into a precision must reject it rather than silently run the wrong codec. + with self.assertRaises(ValueError): + FakeBackend(args(precision="fp8")) + + def test_make_problem_sends_x_and_points_the_oracle_at_semantic_payload(self): + # dispatch_x is always x -- adapters quantize inside dispatch(), where production + # pays it -- and oracle_x is the semantic round-trip, so the two can never drift + # apart the way two independent encode paths could. + backend = FakeBackend(args()) + calls = [] + backend.semantic_payload = lambda value: calls.append(value) or "semantic" + torch = types.ModuleType("torch") + torch.float32, torch.int64 = "float32", "int64" + cast = lambda dtype: f"cast:{dtype}" # noqa: E731 + with mock.patch.dict(sys.modules, {"torch": torch}): + problem = backend.make_problem( + 4, types.SimpleNamespace(to=cast), types.SimpleNamespace(to=cast), "X" + ) + self.assertIs(problem.dispatch_x, problem.x) + self.assertEqual(problem.dispatch_x, "X") + self.assertEqual(problem.oracle_x, "semantic") + self.assertEqual(calls, ["X"]) + + +# ---- from test_roundtrip_staging.py ----------------------------------------------- +class _StagingBackend(ep_backend.EPBackend): + """Records the call order; no device work.""" + + name = "stub" + + def __init__(self, stage_device_work: bool, fp8_consume: str, precision: str = "fp8"): + self.calls: list[str] = [] + self.stage_device_work = stage_device_work + self.fp8_consume = fp8_consume + self.precision = precision + + def create_buffer(self, spec): # pragma: no cover - unused + raise NotImplementedError + + def dispatch(self, problem): + self.calls.append("dispatch") + return types.SimpleNamespace(combine_input=None) + + def stage(self, problem, handle): + self.calls.append("stage") + handle.combine_input = "staged-by-stage" + + def combine(self, problem, handle): + self.calls.append(f"combine({handle.combine_input})") + return handle.combine_input + + def recv_tokens(self, handle): # pragma: no cover - unused + return 0 + + def inspect_dispatch(self, problem, handle): # pragma: no cover - unused + return {} + + def combine_transformed(self, problem, handle, transformed): # pragma: no cover + return transformed + + +class RoundtripStaging(unittest.TestCase): + def test_staged_input_keeps_the_conversion_out_of_the_chain(self): + b = _StagingBackend(stage_device_work=True, fp8_consume="native") + b.run_roundtrip(object(), staged="pre-materialised") + self.assertEqual(b.calls, ["dispatch", "combine(pre-materialised)"]) + self.assertNotIn("stage", b.calls) + + def test_an_unrecognised_consume_mode_fails_instead_of_silently_meaning_native(self): + # The value is read at class-body evaluation, so a typo raises at import -- before + # any measurement -- rather than quietly running the default model and tagging the + # artifact with whatever the typo said. + import importlib + + with mock.patch.dict(os.environ, {"CX_FP8_CONSUME": "dequantize"}): + with self.assertRaises(ValueError): + importlib.reload(ep_backend) + # Restore the module other tests hold references into. + importlib.reload(ep_backend) + self.assertEqual(ep_backend.EPBackend.fp8_consume, "native") + + +class FlashInferCombineModelSwitch(unittest.TestCase): + """Which arithmetic the oracle holds FlashInfer's combine to, selected by wheel version. + + An inverted comparison or a typo'd `_COMBINE_FP32_SINCE` silently swaps the expected + combine for every FlashInfer row -- a wrong-model failure runs 30-90x COMBINE_REL_TOL, + so it reds correct runs rather than passing bad ones, but nothing else in CI sees it. + """ + + def _module(self): + with mock.patch.dict(sys.modules, _stub_modules()): + import importlib + import ep_flashinfer + return importlib.reload(ep_flashinfer) + + def test_the_fp32_boundary_is_exact_and_ordered_by_version_not_by_text(self): + gate = self._module()._wheel_has_fp32_combine + self.assertFalse(gate("0.6.15"), "the wheel below the boundary rounds per level") + self.assertTrue(gate("0.6.16"), "the boundary wheel itself accumulates in FP32") + self.assertTrue(gate("0.6.17")) + # Real version ordering, not a digit scrape: 0.10.0 sorts BELOW 0.6.16 as text. + self.assertTrue(gate("0.10.0")) + + def test_an_unreadable_version_falls_back_to_the_rounding_model(self): + # Asymmetric costs: modelling FP32 against a per-level-rounding kernel can exceed + # COMBINE_REL_TOL and red a correct run, while the opposite error costs a few ulps. + gate = self._module()._wheel_has_fp32_combine + for version in ("0.6.16rc1", "not-a-version", ""): + self.assertFalse(gate(version), f"{version!r} must fall back to the safe model") + + +class NcclLowLatencyLadderClamp(unittest.TestCase): + """The measured ladder is clamped below the receive buffer around an unfixed upstream race.""" + + def _module(self): + with mock.patch.dict(sys.modules, _stub_modules()): + import importlib + import ep_nccl + return importlib.reload(ep_nccl) + + def test_the_ladder_is_clamped_below_the_buffer(self): + # Two separate numbers on purpose: clamping the ladder must not shrink the transport + # footprint, or the rungs that remain quietly measure a smaller receive plane. + m = self._module() + self.assertLess(m._LL_LADDER_CAP, m._LL_BUFFER_CAP) + self.assertLessEqual(m._LL_BUFFER_CAP, 511) + + def _backend(self, module, low_latency): + """A backend far enough along to run create_buffer against the stubs.""" + backend = module.NCCLEPBackend.__new__(module.NCCLEPBackend) + backend._ll = low_latency + backend.world_size, backend.num_local_experts, backend.device = 8, 4, "cuda:0" + backend.args = types.SimpleNamespace(hidden=7168, experts=256, topk=8) + backend._algorithm = "LL" if low_latency else "HT" + backend._bootstrap_comm = lambda: None + backend._comm = object() + module.nccl_ep.Group = types.SimpleNamespace(create=lambda *a, **k: object()) + return backend + + def test_buffer_cap_reports_the_ladder_cap(self): + # Patching the constant and watching the return move proves buffer_cap() reads it, + # which a literal that merely happens to equal it today would not. + module = self._module() + backend = self._backend(module, low_latency=True) + self.assertEqual(backend.buffer_cap(None), module._LL_LADDER_CAP) + with mock.patch.object(module, "_LL_LADDER_CAP", 64): + self.assertEqual(backend.buffer_cap(None), 64) + self.assertIsNone(self._backend(module, low_latency=False).buffer_cap(None)) + + def test_the_receive_is_sized_from_the_buffer_cap_not_the_ladder(self): + # The regression this guards would silently re-baseline every low-latency row: clamping + # the MEASURED ladder must not shrink the receive the remaining rungs are measured + # against. Driven through create_buffer, so it fails on the allocation the kernel gets + # rather than on the shape of the source line that computes it. + module = self._module() + spec = types.SimpleNamespace(max_tokens_per_rank=99) + backend = self._backend(module, low_latency=True) + backend.create_buffer(spec) + self.assertEqual(backend.max_dispatch, module._LL_BUFFER_CAP) + self.assertNotEqual(backend.max_dispatch, module._LL_LADDER_CAP) + with mock.patch.object(module, "_LL_BUFFER_CAP", 512): + sized = self._backend(module, low_latency=True) + sized.create_buffer(spec) + self.assertEqual(sized.max_dispatch, 512) + # Throughput mode is unclamped and keeps taking its size from the ladder spec. + throughput = self._backend(module, low_latency=False) + throughput.create_buffer(spec) + self.assertEqual(throughput.max_dispatch, 99) + + +class RoundtripStagingGate(unittest.TestCase): + """`roundtrip` must mean dispatch -> combine in every row, or it is not comparable: the gate + is `stage_device_work` alone, with `CX_FP8_CONSUME=dequant` as the sole opt-out.""" + + def test_the_gate_truth_table(self): + table = ( + # A real device copy hoists regardless of precision, so MoRI BF16 scale-up and + # FlashInfer BF16 are excluded on the same terms as every fp8 stage... + (True, "native", "bf16", True), + (True, "native", "fp8", True), + # ...a pointer-assignment stage never does: hoisting would hand a low-latency + # backend a view into its double-buffered receive... + (False, "native", "bf16", False), + (False, "native", "fp8", False), + # ...and CX_FP8_CONSUME=dequant restores the inline stage for fp8 only -- a stack + # that really converts between the collectives -- with nothing to model at BF16. + (True, "dequant", "fp8", False), + (True, "dequant", "bf16", True), + ) + for stage_device_work, consume, precision, hoisted in table: + with self.subTest(stage=stage_device_work, consume=consume, precision=precision): + backend = _StagingBackend(stage_device_work, consume, precision) + self.assertEqual(bool(backend.stage_excluded_from_roundtrip), hoisted) + +class WarmStaging(unittest.TestCase): + """Warm-up must not rehearse work the timed region skips: where staging is excluded from the + chain it was the leg's largest single cost (~247us x 32 iters x every component x trial).""" + + @staticmethod + def _warm(backend, count, **kwargs): + # `warm` imports torch for one synchronize; a stub keeps this runnable without a GPU. + fake = types.ModuleType("torch") + fake.cuda = types.SimpleNamespace(synchronize=lambda: None) + saved = sys.modules.get("torch") + sys.modules["torch"] = fake + try: + backend.warm(types.SimpleNamespace(), count, **kwargs) + finally: + if saved is None: + del sys.modules["torch"] + else: + sys.modules["torch"] = saved + + def test_stages_once_when_the_chain_excludes_staging(self): + b = _StagingBackend(stage_device_work=True, fp8_consume="native") + self._warm(b, 5) + self.assertEqual(b.calls.count("dispatch"), 5) + self.assertEqual(b.calls.count("stage"), 1) + # Every later iteration still hands combine the staged payload, not a stale None. + self.assertEqual(b.calls.count("combine(staged-by-stage)"), 5) + +# The chained-period staging contract lives in tests/test_chain_period.py, which asserts it per +# sibling chain with window values. + + +# ---- from test_ep_nccl_handle.py -------------------------------------------------- +def _stub_modules(): + """Fake torch / nccl modules so `import ep_nccl` succeeds without the benchmark image.""" + torch = types.ModuleType("torch") + torch.bfloat16 = "bfloat16" + torch.int32 = "int32" + torch.float32 = "float32" + torch.int64 = "int64" + torch.empty = lambda *a, **k: types.SimpleNamespace(shape=a[0] if a else ()) + torch.empty_like = lambda *a, **k: types.SimpleNamespace() + torch.zeros = lambda *a, **k: types.SimpleNamespace(item=lambda: 7) + torch.cuda = types.SimpleNamespace(synchronize=lambda: None) + dist = types.ModuleType("torch.distributed") + torch.distributed = dist + + ep = types.ModuleType("nccl.ep") + for name in ( + "Algorithm", "CombineConfig", "CombineInputs", "CombineOutputs", "DispatchConfig", + "DispatchInputs", "DispatchOutputs", "GroupConfig", "HandleConfig", "Layout", + "LayoutInfo", "Tensor", + ): + setattr(ep, name, type(name, (), {"__init__": lambda self, *a, **k: None})) + ep.Algorithm = types.SimpleNamespace(LOW_LATENCY="LL", HIGH_THROUGHPUT="HT") + ep.Layout = types.SimpleNamespace(EXPERT_MAJOR="EM", FLAT="FLAT") + core = types.ModuleType("nccl.core") + pkg = types.ModuleType("nccl") + pkg.ep, pkg.core = ep, core + return { + "torch": torch, "torch.distributed": dist, + "nccl": pkg, "nccl.ep": ep, "nccl.core": core, + } + + +sys.path[:0] = [str(ROOT), str(ROOT / "bench")] + +# Import ep_nccl against the stubs, then withdraw them: a fake torch left in sys.modules makes +# genuinely torch-dependent modules (test_runtime, test_ll_oracle) error instead of skipping. +with mock.patch.dict(sys.modules, _stub_modules()): + import ep_nccl # noqa: E402 + + sys.modules.pop("ep_nccl", None) + + +class FakeHandle: + """Records every rebind so the tests can assert on the collective call pattern.""" + + def __init__(self): + self.updates = [] + self.destroyed = False + + def update(self, topk_idx, *, layout_info=None, stream=None): + self.updates.append((topk_idx, layout_info)) + + def destroy(self): + self.destroyed = True + + +class FakeGroup: + def __init__(self): + self.created = 0 + self.handle = FakeHandle() + + def create_handle(self, layout, topk_idx, *, layout_info=None, config=None, stream=None): + self.created += 1 + return self.handle + + +def backend(ll=True): + """An NCCLEPBackend with just the fields _ensure_handle touches (no __init__, no GPU).""" + b = object.__new__(ep_nccl.NCCLEPBackend) + b._ll = ll + b._layout = "EM" if ll else "FLAT" + b._handle = None + b._bound = None + b._ep_group = FakeGroup() + b.device = "cuda:0" + b.num_local_experts = 4 + b.args = types.SimpleNamespace(hidden=16) + b._t = lambda x: x + b._stream = lambda: 0 + # create_buffer always runs before the first _ensure_handle, so the HT receive plane exists + # by then; a list stands in for the tensor because `_t` is identity here. + b._recv_x = list(range(64)) + return b + + +def problem(T): + return types.SimpleNamespace( + T=T, dispatch_x=f"x{T}", topk_idx=f"idx{T}", topk_weights=f"w{T}" + ) + + +class TestSingleHandle(unittest.TestCase): + def test_one_handle_across_many_shapes(self): + """Nine ladder rungs must still produce exactly one create_handle.""" + b = backend() + for T in (1, 2, 4, 8, 16, 32, 64, 128, 256): + b._ensure_handle(problem(T)) + self.assertEqual(b._ep_group.created, 1) + + def test_ll_gate_wrapper_is_built_once_per_handle(self): + """LL applies the gate in combine, so its weights wrapper must be cached: building one + per timed combine puts a torch resolve and an np.asarray inside `time_us`.""" + ll = backend(ll=True) + pa = problem(1) + h = ll._ensure_handle(pa) + self.assertTrue(hasattr(h, "combine_weights_t")) + self.assertEqual(h.combine_weights_t, "w1") + # Re-entering the same problem reuses the handle and therefore the wrapper. + self.assertIs(ll._ensure_handle(pa).combine_weights_t, h.combine_weights_t) + + ht = backend(ll=False) + self.assertFalse(hasattr(ht._ensure_handle(problem(1)), "combine_weights_t")) + + def test_ht_combine_input_is_sliced_to_the_received_count(self): + """HT combine's staging copy is sized by the tensor it is handed: the whole ladder-max + receive plane put a rung-independent floor under it. LL keeps the full padded plane.""" + b = backend(ll=False) + h = b._ensure_handle(problem(1)) + # 7 is what the stubbed `torch.zeros(...).item()` reports as the received count. + self.assertEqual(h.count, 7) + self.assertEqual(h.combine_in_t, list(range(7))) + self.assertLess(len(h.combine_in_t), len(b._recv_x)) + + ll = backend(ll=True) + ll_h = ll._ensure_handle(problem(1)) + self.assertFalse(hasattr(ll_h, "combine_in_t")) + +if __name__ == "__main__": + unittest.main() diff --git a/experimental/CollectiveX/tests/test_bandwidth.py b/experimental/CollectiveX/tests/test_bandwidth.py deleted file mode 100644 index aed9c4b5e..000000000 --- a/experimental/CollectiveX/tests/test_bandwidth.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 -"""Math tests for the bandwidth consumer (unit conversion, alpha/beta fit, fit gating).""" -from __future__ import annotations - -import sys -import unittest -from pathlib import Path - -sys.path[:0] = [str(Path(__file__).resolve().parents[1])] - -import bandwidth # noqa: E402 - -COMPONENTS = bandwidth.COMPONENTS - - -def _row(tokens, nbytes, latency, passed=True): - """A measurement row. `latency` is a scalar, or a per-component dict whose None marks - that component unavailable.""" - lat = latency if isinstance(latency, dict) else dict.fromkeys(COMPONENTS, latency) - return { - "tokens_per_rank": tokens, - "components": {c: {"percentiles_us": None if lat[c] is None else { - "p50": lat[c], "p90": lat[c], "p95": lat[c], "p99": lat[c] * 2.0}} - for c in COMPONENTS}, - "byte_provenance": {c: {"total_logical_bytes": nbytes} for c in COMPONENTS}, - "correctness": {"passed": passed}, - "routing": {"locality": {"cross_node_fraction": 0.5}}, - } - - -def _doc(rows, ep=2, mode="normal"): - case = {"ep": ep, "backend": "deepep-v2", "precision": "bf16", "phase": "decode", - "mode": mode, "suite": "s", "routing": "uniform"} - return { - "generated_at": "2026-07-25T22:12:55.760511+00:00", - "identity": {"attempt_ordinal": 1, "allocation_factors": {"run_id": "30177021271"}, - "case_factors": {"sku": "h100", "case": case}}, - "measurement": {"rows": rows}, - "outcome": {"status": "success"}, - } - - -def _linear(pairs, passed=True): - """Rows exactly on latency = 10us + bytes * 2e-6, i.e. alpha=10, beta_agg=500 GB/s.""" - return [_row(t, b, 10.0 + b * 2e-6, passed) for t, b in pairs] - - -LADDER = ((8, 1e6), (16, 2e6), (32, 3e6)) - - -class BandwidthMath(unittest.TestCase): - def test_algbw_per_gpu(self): - # 1e9 bytes in 1000us = 1e12 B/s = 1000 GB/s aggregate; /ep(2) = 500 per GPU. - self.assertAlmostEqual(bandwidth._algbw_per_gpu(1e9, 1000.0, 2), 500.0) - self.assertIsNone(bandwidth._algbw_per_gpu(1e9, 0.0, 2)) - - def test_fit_recovers_alpha_beta(self): - fit = bandwidth.fit_alpha_beta(_doc(_linear(LADDER)), "dispatch") - self.assertAlmostEqual(fit.alpha_us, 10.0, places=4) - self.assertAlmostEqual(fit.beta_gbps, 250.0, places=4) # 500 aggregate / ep(2) - self.assertAlmostEqual(fit.r2, 1.0, places=6) - self.assertEqual(fit.points, 3) - self.assertTrue(fit.beta_is_reliable) - - def test_fit_is_none_when_undefensible(self): - flat = [_row(t, b, 12.0) for t, b in LADDER] # slope <= 0 - self.assertIsNone(bandwidth.fit_alpha_beta(_doc(_linear(LADDER[:2])), "dispatch")) - self.assertIsNone(bandwidth.fit_alpha_beta(_doc(flat), "dispatch")) - - def test_noisy_ladder_withholds_beta(self): - # A positive slope through noise still fits; printing its beta once produced a - # physically impossible 1018 GB/s per GPU on a B200 at R2 = 0.29. - rows = [_row(t, b, lat) for t, b, lat in ( - (8, 1e6, 300.0), (16, 2e6, 40.0), (32, 3e6, 260.0), - (64, 4e6, 60.0), (128, 5e6, 320.0))] - fit = bandwidth.fit_alpha_beta(_doc(rows), "dispatch") - self.assertLess(fit.r2, bandwidth.FIT_MIN_R2) - self.assertFalse(fit.beta_is_reliable) - out = bandwidth.render([_doc(rows)]) - self.assertIn("beta=unreliable", out) - self.assertNotIn("GB/s alpha", out) # no number presented as measured - - def test_latency_bound_ladder_withholds_beta_despite_high_r2(self): - # Real data gave beta = 3763 GB/s at R2 = 0.92: a near-zero slope explodes beta while - # the line still fits, so R2 cannot catch it — the transfer-share gate must. - rows = [_row(t, b, 500.0 + b * 1e-9) - for t, b in LADDER + ((64, 4e6), (128, 5e6))] - fit = bandwidth.fit_alpha_beta(_doc(rows), "dispatch") - self.assertGreater(fit.r2, bandwidth.FIT_MIN_R2) - self.assertLess(fit.bandwidth_share, bandwidth.FIT_MIN_BANDWIDTH_SHARE) - self.assertFalse(fit.beta_is_reliable) - self.assertIn("not bandwidth-bound", bandwidth._format_fit("dispatch", fit)) - - def test_alpha_marked_only_when_extrapolated(self): - prefill = bandwidth.fit_alpha_beta( # starts at T=1024: intercept is extrapolated - _doc(_linear(((1024, 1e9), (2048, 2e9), (4096, 4e9), (8192, 8e9)))), "dispatch") - decode = bandwidth.fit_alpha_beta( # reaches near zero bytes: alpha stands - _doc(_linear(((1, 1e5), (64, 6.4e6), (512, 5.12e7)))), "dispatch") - self.assertTrue(prefill.alpha_extrapolated) - self.assertFalse(decode.alpha_extrapolated) - self.assertIn("*", bandwidth._format_fit("dispatch", prefill)) - self.assertNotIn("*", bandwidth._format_fit("dispatch", decode)) - - def test_gate_failed_rung_excluded_from_fit_and_marked(self): - rows = _linear(LADDER) + [_row(64, 4e6, 999.0, passed=False)] - fit = bandwidth.fit_alpha_beta(_doc(rows), "dispatch") - self.assertEqual((fit.points, fit.excluded_rows), (3, 1)) - self.assertAlmostEqual(fit.beta_gbps, 250.0, places=4) # the corrupt rung didn't steer it - out = bandwidth.render([_doc(rows)]) - self.assertIn("[correctness FAILED]", out) - self.assertIn("excluded 1 gate-failed rung", out) - - def test_render_marks_unavailable_and_separates_attempts(self): - unavailable = [_row(t, b, {"dispatch": None, "combine": 5.0, "roundtrip": 6.0}) - for t, b in LADDER] - out = bandwidth.render([_doc(unavailable)]) - self.assertIn("dispatch=n/a", out) - self.assertIn("xnode= 50%", out) - second = _doc(_linear(LADDER)) - second["identity"]["attempt_ordinal"] = 2 - second["generated_at"] = "2026-07-25T23:00:00.000000+00:00" - out = bandwidth.render([_doc(_linear(LADDER)), second]) - self.assertIn("attempt 1", out) - self.assertIn("attempt 2", out) - self.assertIn("run 30177021271", out) - - -if __name__ == "__main__": - unittest.main() diff --git a/experimental/CollectiveX/tests/test_chain.py b/experimental/CollectiveX/tests/test_chain.py new file mode 100644 index 000000000..5af796bfe --- /dev/null +++ b/experimental/CollectiveX/tests/test_chain.py @@ -0,0 +1,976 @@ +#!/usr/bin/env python3 +"""The chained pair period end to end: the timing primitive, its publication through run_sweep, and the headline it feeds. + +Two torch doubles live here on purpose. The trace_* family logs record()/sync()/all_reduce against a controllable clock, so event PLACEMENT is assertable; the value_* family carries real tensor arithmetic, so the published NUMBERS are. Merging them would be more complex than either.""" +from __future__ import annotations + +import contextlib +import io +import sys +import types +import unittest +from pathlib import Path +from unittest import mock +import copy +import json +import os +import statistics +import tempfile +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT), str(ROOT / "bench")] +sys.path[:0] = [str(ROOT)] + +import ep_backend # noqa: E402 +import ep_harness # noqa: E402 +import summarize # noqa: E402 + + +# ---- from test_chain_period.py ---------------------------------------------------- +# Per-operation device cost in the stub clock (ms). Distinct primes so that any window +# reports a sum unique to the operations it actually brackets. +DISPATCH_MS = 3.0 +STAGE_MS = 7.0 +COMBINE_MS = 5.0 + + +class _Clock: + """Stub device clock; only the fake backend's operations advance it.""" + + def __init__(self): + self.now_ms = 0.0 + + def advance(self, ms): + self.now_ms += ms + + +class _TraceEvent: + """torch.cuda.Event stand-in; logs record() into the shared call trace so placement is + assertable.""" + + def __init__(self, clock, log=None): + self._clock = clock + self._log = log + self.t = None + + def record(self, *_args, **_kwargs): + self.t = self._clock.now_ms + if self._log is not None: + self._log.append("record") + + def elapsed_time(self, other): + if self.t is None or other.t is None: + raise AssertionError("elapsed_time on an event that was never recorded") + return other.t - self.t + + def synchronize(self, *_args, **_kwargs): + pass + + def query(self): + return True + + +class _TraceTensor: + """Absorbs whatever a tensor is asked to do.""" + + def __getattr__(self, _name): + return lambda *args, **kwargs: self + + +@contextlib.contextmanager +def trace_torch(clock, log): + """Install a stub `torch`/`torch.distributed` that logs the calls this contract is about.""" + tensor = lambda *args, **kwargs: _TraceTensor() # noqa: E731 + dist = types.SimpleNamespace( + all_reduce=lambda *args, **kwargs: log.append("all_reduce"), + barrier=lambda *args, **kwargs: log.append("dist_barrier"), + is_initialized=lambda: True, + get_rank=lambda *args, **kwargs: 0, + get_world_size=lambda *args, **kwargs: 2, + ReduceOp=types.SimpleNamespace(SUM="sum", MAX="max", MIN="min"), + ) + torch = types.SimpleNamespace( + cuda=types.SimpleNamespace( + Event=lambda *args, **kwargs: _TraceEvent(clock, log), + synchronize=lambda *args, **kwargs: log.append("sync"), + current_stream=lambda *args, **kwargs: types.SimpleNamespace( + synchronize=lambda: log.append("sync") + ), + ), + distributed=dist, + zeros=tensor, ones=tensor, empty=tensor, full=tensor, tensor=tensor, + float32="float32", float64="float64", bfloat16="bfloat16", int32="int32", + ) + with mock.patch.dict(sys.modules, {"torch": torch, "torch.distributed": dist}): + yield torch + + +class _Combined: + """Just enough combined-output tensor for the chain's final-output capture.""" + + def __init__(self, value): + self.value = value + self.cloned = False + + def clone(self): + detached = _Combined(self.value) + detached.cloned = True + return detached + + +class _ChainBackend(ep_backend.EPBackend): + """Records the call order and charges each operation a fixed slice of the stub clock.""" + + name = "chain-stub" + + def __init__(self, stage_device_work=True, fp8_consume="native", precision="fp8", + dispatch_schedule=None): + self.calls: list[str] = [] + self.consumed: list = [] + self.clock = _Clock() + self.stage_device_work = stage_device_work + self.fp8_consume = fp8_consume + self.precision = precision + self.device = "cpu" + self.rank = 0 + self.world_size = 2 + # Per-dispatch cost overrides, consumed in order; the constant cost applies after. + self._dispatch_schedule = list(dispatch_schedule or []) + + def create_buffer(self, spec): # pragma: no cover - unused + raise NotImplementedError + + def dispatch(self, problem): + self.calls.append("dispatch") + cost = self._dispatch_schedule.pop(0) if self._dispatch_schedule else DISPATCH_MS + self.clock.advance(cost) + return types.SimpleNamespace(combine_input=None) + + def stage(self, problem, handle): + self.calls.append("stage") + self.clock.advance(STAGE_MS) + handle.combine_input = "staged-by-stage" + + def combine(self, problem, handle): + self.calls.append("combine") + self.consumed.append(handle.combine_input) + self.clock.advance(COMBINE_MS) + return _Combined(handle.combine_input) + + def recv_tokens(self, handle): + return 0 + + def inspect_dispatch(self, problem, handle): # pragma: no cover - unused + return {} + + def combine_transformed(self, problem, handle, transformed): # pragma: no cover + return transformed + + +def new_problem(): + """A problem the backend can hang cached state on -- `warm` caches recv_tokens there.""" + return types.SimpleNamespace() + + +def timed_tail(calls, iters, per_pair): + """The period chain's ops: the last `per_pair * iters` op entries, records and syncs removed.""" + trace = [entry for entry in calls if entry != "record"] + while trace and trace[-1] in ("sync", "all_reduce", "dist_barrier"): + trace.pop() + return trace[-per_pair * iters:] + + +def ops_only(calls): + """Just the backend operations, in order.""" + return [entry for entry in calls if entry in ("dispatch", "stage", "combine")] + + +def chain_sections(calls): + """(floors_chain, period_chain) raw-trace slices; the chains sit between the last three syncs.""" + sync_idx = [i for i, entry in enumerate(calls) if entry == "sync"] + end_period, end_floors = sync_idx[-1], sync_idx[-2] + start_floors = sync_idx[-3] + 1 if len(sync_idx) >= 3 else 0 + return calls[start_floors:end_floors], calls[end_floors + 1:end_period] + + +class ChainedPairPeriod(unittest.TestCase): + def test_the_loop_is_free_running_dispatch_combine_pairs(self): + # A host sync inside the loop drains the GPU and turns the period back into a sequence of + # drained roundtrips; a cross-rank call between pairs re-aligns the ranks and buys back + # the very stagger the chain exists to amortise. Neither may appear. + iters = 6 + backend = _ChainBackend() + with trace_torch(backend.clock, backend.calls): + backend.benchmark_chain(new_problem(), 0, iters, 2) + self.assertEqual( + timed_tail(backend.calls, iters, 2), ["dispatch", "combine"] * iters + ) + self.assertNotIn("all_reduce", backend.calls) + self.assertNotIn("dist_barrier", backend.calls) + + def test_a_hoisted_stage_runs_once_and_stays_out_of_every_pair_window(self): + # The conversion is materialised once, untimed, so the pair is dispatch -> combine in + # both chains. The dequant hatch is fp8-only, so a bf16 row keeps its hoist regardless. + for precision, consume in (("bf16", "native"), ("fp8", "native"), ("bf16", "dequant")): + with self.subTest(precision=precision, consume=consume): + iters = 6 + backend = _ChainBackend( + stage_device_work=True, fp8_consume=consume, precision=precision + ) + self.assertTrue(backend.stage_excluded_from_roundtrip) + with trace_torch(backend.clock, backend.calls): + series = backend.benchmark_chain(new_problem(), 0, iters, 2) + self.assertEqual(backend.calls.count("stage"), 1) + self.assertEqual(backend.consumed, ["staged-by-stage"] * (2 * iters + 1)) + floors, period = chain_sections(backend.calls) + self.assertEqual(ops_only(floors), ["dispatch", "combine"] * iters) + self.assertEqual(ops_only(period), ["dispatch", "combine"] * iters) + # The staged cost is absent from the pair window, not merely from the trace. + for value in series["pair"]: + self.assertAlmostEqual(value, (DISPATCH_MS + COMBINE_MS) * 1000.0) + for value in series["start_to_start"]: + self.assertAlmostEqual(value, (DISPATCH_MS + COMBINE_MS) * 1000.0) + for value in series["dispatch"]: + self.assertAlmostEqual(value, DISPATCH_MS * 1000.0) + for value in series["combine"]: + self.assertAlmostEqual(value, COMBINE_MS * 1000.0) + + def test_returns_one_sample_per_kept_iteration(self): + for iters, drop in ((8, 0), (8, 2), (6, 5)): + with self.subTest(iters=iters, drop=drop): + backend = _ChainBackend() + with trace_torch(backend.clock, backend.calls): + series = backend.benchmark_chain(new_problem(), 0, iters, drop) + self.assertEqual( + sorted(series), + ["combine", "combined", "dispatch", "pair", "start_to_start"], + ) + for key in ("pair", "dispatch", "combine"): + self.assertEqual(len(series[key]), iters - drop) + # Start-to-start is a difference series: one fewer than the kept pairs. + self.assertEqual(len(series["start_to_start"]), max(iters - drop - 1, 0)) + + def test_the_dropped_iterations_are_the_head_of_each_chain(self): + # `drop` discards pipeline fill, so it must cut the head of both chains -- the period + # chain refills after the inter-chain synchronize. + iters, drop = 6, 2 + slow_head = [50.0] * drop + [DISPATCH_MS] * (iters - drop) + backend = _ChainBackend( + stage_device_work=False, fp8_consume="native", precision="bf16", + dispatch_schedule=slow_head * 2, # floors chain runs first, then the period chain + ) + with trace_torch(backend.clock, backend.calls): + series = backend.benchmark_chain(new_problem(), 0, iters, drop) + self.assertEqual(len(series["dispatch"]), iters - drop) + for value in series["dispatch"]: + self.assertAlmostEqual(value, DISPATCH_MS * 1000.0) + for value in series["pair"]: + self.assertAlmostEqual( + value, (DISPATCH_MS + STAGE_MS + COMBINE_MS) * 1000.0 + ) + + +class EventPlacement(unittest.TestCase): + """Which events each sibling chain may carry. The stub charges host work nothing, so these + assert record placement in the trace rather than window values.""" + + def _sections(self, **backend_kwargs): + iters = 4 + backend = _ChainBackend(**backend_kwargs) + with trace_torch(backend.clock, backend.calls): + backend.benchmark_chain(new_problem(), 0, iters, 1) + floors, period = chain_sections(backend.calls) + return iters, floors, period + + def test_the_period_pairs_carry_only_the_outer_events(self): + # One record before the dispatch, one after the combine, nothing between: both records' + # host cost lands in the inter-pair gap, outside the published window. + iters, _, period = self._sections( + stage_device_work=False, fp8_consume="native", precision="bf16" + ) + self.assertEqual( + period, + ["record", "dispatch", "stage", "combine", "record"] * iters, + ) + +class ChainBudgetGate(unittest.TestCase): + """A budget that could publish a degenerate chain must stop the leg first: zero kept pairs + serialises as "unavailable", indistinguishable from a backend that cannot chain at all, and + a single kept pair would publish a period whose health scalars are silently unavailable.""" + + @staticmethod + def _args(**updates): + values = dict( + mode="normal", iters=8, trials=256, warmup=32, + chain_iters=128, chain_trials=4, chain_drop=16, + ) + values.update(updates) + return types.SimpleNamespace(**values) + + def _gate(self, **updates): + """rc and rank-0 output; None stands in for every device-side argument.""" + with contextlib.redirect_stdout(io.StringIO()) as out: + rc = ep_harness.run_sweep(self._args(**updates), None, None, None, None, 0, 1) + return rc, out.getvalue() + + def test_an_unusable_chain_budget_fails_closed(self): + for label, updates in ( + ("no iterations", dict(chain_iters=0)), + ("no trials", dict(chain_trials=0)), + ("drop swallows every pair", dict(chain_iters=8, chain_drop=8)), + ("drop exceeds the chain", dict(chain_iters=8, chain_drop=9)), + # A single kept pair has an empty start-to-start series: interpair gap and settle + # drift would silently publish as "unavailable" health for a measured period. + ("drop leaves a single pair", dict(chain_iters=8, chain_drop=7)), + ("negative drop", dict(chain_drop=-1)), + ): + with self.subTest(budget=label): + rc, output = self._gate(**updates) + self.assertEqual(rc, 2) + self.assertIn("chain", output) + + def test_the_chain_gate_did_not_displace_the_fresh_entry_one(self): + # Both budgets are checked and each names its own fields, so the failure says which + # profile field to fix. + rc, output = self._gate(iters=0) + self.assertEqual(rc, 2) + self.assertIn("iters/trials/warmup", output) + + +class ChainComponentContract(unittest.TestCase): + """What the driven sweep (test_run_sweep_chain.py) does not exercise: the constants a + consumer imports by name, and `_component` on the two paths no chain row takes.""" + + def test_the_origin_constants_carry_the_published_values(self): + self.assertEqual(ep_harness.CHAIN_PERIOD_ORIGIN, "chained-median") + self.assertEqual(ep_harness.CHAIN_FLOOR_ORIGIN, "chained-cross-rank-min") + + def test_an_overridden_origin_leaves_the_rest_of_the_component_alone(self): + # Every pre-chain row also flows through `_component`, so omitting the override must + # reproduce the old strings exactly or the chain reclassifies unrelated rows. + percentiles = {"p50": 1.0, "p90": 2.0, "p95": 3.0, "p99": 4.0} + self.assertEqual(ep_harness._component(percentiles, 3)["origin"], "measured") + self.assertEqual( + ep_harness._component(percentiles, 0, derived=True)["origin"], + "derived-percentile-sum", + ) + self.assertIsNone(ep_harness._component(None, 0)["origin"]) + + overridden = ep_harness._component(percentiles, 3, origin="chained-median") + self.assertEqual(overridden["origin"], "chained-median") + self.assertEqual(overridden["availability"], "measured") + self.assertEqual(overridden["percentiles_us"], percentiles) + self.assertEqual(overridden["sample_count"], 3) + + +# ---- from test_run_sweep_chain.py ------------------------------------------------- +LADDER = [4, 8] +CHAIN_ITERS, CHAIN_DROP, CHAIN_TRIALS = 8, 2, 2 +KEPT_PER_TRIAL = CHAIN_ITERS - CHAIN_DROP +# What the stub backend reports for every chained iteration, distinct so a published number is +# traceable to the op it came from; start_to_start sits a fixed GAP above the pair window. +PAIR_US, DISPATCH_FLOOR_US, COMBINE_FLOOR_US = 50.0, 20.0, 25.0 +GAP_US, DRIFT_US = 4.0, 8.0 +UNAVAILABLE = { + "availability": "unavailable", "origin": None, "percentiles_us": None, "sample_count": 0, +} + + +class _ValueTensor: + """Enough tensor for the reductions run_sweep performs: gather, stack, median/max/min, sub.""" + + def __init__(self, data): + self.data = data + + def tolist(self): + return self.data + + def item(self): + return self.data[0] + + def clone(self): + return _ValueTensor(copy.deepcopy(self.data)) + + def to(self, *args, **kwargs): + return self + + def __iter__(self): + return iter(self.data) + + def __sub__(self, other): + return _ValueTensor([a - b for a, b in zip(self.data, other.data)]) + + def _reduce(self, fn): + columns = ( + [[row[i] for row in self.data] for i in range(len(self.data[0]))] + if self.data else [] + ) + return SimpleNamespace(values=_ValueTensor([fn(column) for column in columns])) + + def median(self, dim=0): + return self._reduce(statistics.median) + + def max(self, dim=0): + return self._reduce(max) + + def min(self, dim=0): + return self._reduce(min) + + @property + def shape(self): + return (len(self.data),) + + +class _ValueEvent: + clock = [0.0] + + def __init__(self, enable_timing=False): + self.t = None + + def record(self): + _ValueEvent.clock[0] += 1.0 + self.t = _ValueEvent.clock[0] + + def elapsed_time(self, other): + return (other.t - self.t) / 1000.0 + + +class _FakeDist: + """World size 1, so every collective is the identity and the artifact is this rank's view.""" + + ReduceOp = SimpleNamespace(MAX="max", MIN="min", SUM="sum") + + @staticmethod + def get_world_size(): + return 1 + + @staticmethod + def get_rank(): + return 0 + + @staticmethod + def all_reduce(tensor, op=None): + return None + + @staticmethod + def all_gather(out, local): + out[0].data = list(local.data) + + @staticmethod + def broadcast(tensor, src=0): + return None + + +def value_torch(): + torch = types.ModuleType("torch") + torch.float64, torch.int64, torch.bfloat16 = "f64", "i64", "bf16" + torch.cuda = SimpleNamespace(synchronize=lambda: None, Event=_ValueEvent) + torch.tensor = lambda values, device=None, dtype=None: _ValueTensor(list(values)) + torch.empty_like = lambda x: _ValueTensor(list(x.data)) + torch.stack = lambda xs: _ValueTensor([x.data for x in xs]) + torch.equal = lambda a, b: a.data == b.data + torch.zeros = lambda n, device=None: _ValueTensor([0.0] * n) + torch.distributed = SimpleNamespace(all_reduce=lambda x: None) + return torch + + +def fake_routing(): + routing = types.ModuleType("routing") + routing.routing_stats = lambda idx, experts, per_rank: { + "empty_expert_count": 0, "empty_rank_count": 0, "expert_assignment_rank_cv": 0.0, + "expert_assignments_per_rank": [8], "expert_load_cv": 0.0, "expert_load_max": 1, + "expert_load_mean": 1.0, "expert_load_min": 1, "fanout_histogram": {}, "fanout_max": 1, + "fanout_mean": 1.0, "fanout_min": 1, "hotspot_ratio": 1.0, + "payload_copies_per_rank": [1], "payload_rank_cv": 0.0, "routed_copies": 8, + } + routing.routing_locality = lambda *args, **kwargs: 1.0 + return routing + + +class _SweepBackend(ep_backend.EPBackend): + """Constant-cost backend; every timed call returns a value unique to what it measures.""" + + name = "stub" + maturity = "candidate" + + def __init__(self): + self.mode = "normal" + self.precision = "bf16" + self.stage_device_work = False + self.fp8_consume = "native" + self.device = "cuda:0" + self.events = [] + + def make_inputs(self, args): + spec = ep_backend.WorkloadSpec( + ep_size=1, experts_per_rank=256, cap=None, dropped=[], + max_tokens_per_rank=max(LADDER), ladder=list(LADDER), + ) + for tokens in spec.ladder: + spec.points[tokens] = ep_backend.RankInputs( + tokens_per_rank=tokens, topk_idx=_ValueTensor([0]), + topk_weights=_ValueTensor([1.0]), activations=_ValueTensor([1.0]), + global_idx=_ValueTensor([0]), global_weights=_ValueTensor([1.0]), + ) + return spec + + def make_problem(self, T, idx, weights, x): + return SimpleNamespace(T=T, x=x, dispatch_x=x, topk_idx=idx, topk_weights=weights) + + def create_buffer(self, spec): + return None + + def warm(self, problem, count, stage_every=False): + return None + + def benchmark_component(self, component, problem, warmup, iters): + return [10.0] * iters + + def benchmark_chain(self, problem, warmup, iters, drop): + self.events.append(("chain", problem.T)) + kept = iters - drop + return { + "pair": [PAIR_US] * kept, + "start_to_start": [PAIR_US + GAP_US] * (kept - 1), + "dispatch": [DISPATCH_FLOOR_US] * kept, + "combine": [COMBINE_FLOOR_US] * kept, + "combined": f"chained-{problem.T}", + } + + def dispatch(self, problem): + # Only the harness's drained reference pair reaches this: warm is a no-op, the + # components are constants, and the oracle is mocked out in `_sweep`. + self.events.append(("drained", problem.T)) + return SimpleNamespace(combine_input=None) + + def stage(self, problem, handle): + return None + + def combine(self, problem, handle): + return f"drained-{problem.T}" + + def recv_tokens(self, handle): + return 8 + + def inspect_dispatch(self, problem, handle): + return {} + + def combine_transformed(self, problem, handle, transformed): + return transformed + + +def make_args(out): + return SimpleNamespace( + mode="normal", precision="bf16", phase="decode", + tokens_ladder=" ".join(map(str, LADDER)), + hidden=7168, topk=8, experts=256, routing="uniform", + case_id="sku-stub-deepseek-v3-normal-decode-ep1-uniform-bf16", + suite="ep-core", workload_name="deepseek-v3", seed=67, version=1, + warmup=2, iters=4, trials=2, + chain_iters=CHAIN_ITERS, chain_trials=CHAIN_TRIALS, chain_drop=CHAIN_DROP, + runner="sku", topology_class="tc", transport="nvlink", scope="scale-up", + scale_up_transport="nvlink", scale_out_transport="", gpus_per_node=1, + scale_up_domain=1, out=str(out), runtime={}, image="", git_run=None, + ) + + +def phases_by_index(oracle_count, points): + """Which pass each oracle call belongs to, by position: Pass 1 opens and Pass 3 closes with one + per point, the middle is the chained gate.""" + return ( + ["pre"] * points + + ["chain"] * (oracle_count - 2 * points) + + ["post"] * points + ) + + +def _sweep(fail_indices, error_indices, chain_error, backend_factory=None, + chain_output_ok=True): + """One full run_sweep against the stubs; failures scripted by oracle call index.""" + backend = (backend_factory or _SweepBackend)() + events = backend.events + oracle_calls = [] + output_checks = [] + + def fake_oracle(torch_, routing_, backend_, problem, *rest): + index = len(oracle_calls) + events.append(("oracle", problem.T)) + oracle_calls.append((index, problem.T, (problem, *rest))) + passed = index not in fail_indices + return ep_harness._oracle_report( + passed=passed, + receive_count=8, + max_elementwise_relative_error=chain_error if index in error_indices else 0.0, + checks=dict.fromkeys(ep_harness._ORACLE_CHECKS, passed), + ) + + def fake_output_match(chained, drained): + output_checks.append((chained, drained)) + return chain_output_ok, 0.0 if chain_output_ok else 1.0 + + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) / "result.json" + stdout = io.StringIO() + with mock.patch.dict(sys.modules, {"routing": fake_routing()}), \ + mock.patch.dict(os.environ, {"COLLX_ATTEMPT_ID": "1"}), \ + mock.patch.object(ep_harness, "_run_expert_oracle", fake_oracle), \ + mock.patch.object(ep_harness, "_chain_output_matches", fake_output_match), \ + contextlib.redirect_stdout(stdout): + rc = ep_harness.run_sweep( + make_args(out), backend, value_torch(), _FakeDist(), "cuda:0", 0, 1 + ) + doc = json.loads(out.read_text()) + return SimpleNamespace( + rc=rc, doc=doc, rows=doc["measurement"]["rows"], events=events, + oracle_calls=oracle_calls, output_checks=output_checks, + stdout=stdout.getvalue(), backend=backend, + phases=phases_by_index(len(oracle_calls), len(LADDER)), + ) + + +def drive(*, fail_phases=(), chain_error=0.0, backend_factory=None, chain_output_ok=True): + """Run the sweep, optionally failing every oracle of a given pass; a clean probe run first + learns the oracle call count, so failures are selected by phase rather than by hardcoded index.""" + probe = _sweep(frozenset(), frozenset(), 0.0, backend_factory) + if not fail_phases and not chain_error and chain_output_ok: + return probe + selected = lambda wanted: frozenset( # noqa: E731 + index for index, phase in enumerate(probe.phases) if phase in wanted + ) + return _sweep( + selected(set(fail_phases)), + selected({"chain"}) if chain_error else frozenset(), chain_error, backend_factory, + chain_output_ok=chain_output_ok, + ) + + +class ChainedRegimeOracleGate(unittest.TestCase): + """The published regime has to be the gated one: Passes 1 and 3 only ever check drained calls, + so a backend that corrupts only under free-running pairs would present as the suite's fastest.""" + + def test_a_chain_output_mismatch_reds_the_case_on_its_own(self): + # The two chained verdicts are independent: a chain whose own final output is wrong + # reds the case even when the state it leaves behind still passes a fresh oracle -- + # exactly the shape a stale-parity/aliased-signal defect presents. + # + # This check was briefly demoted to reporting-only (2026-08-07) on the theory that its + # tolerance was too tight for FP8. Probe 31180411148 measured the magnitude and + # falsified that: bf16 differs by exactly 0.0, FP8 by 1000x-2966x the tolerance. Re-armed + # on that number. If it is ever demoted again, demote it on a MEASURED magnitude too -- + # the verdict alone is what justified the wrong call both times. + run = drive(chain_output_ok=False) + self.assertEqual(run.rc, 3) + self.assertEqual(run.doc["outcome"]["status"], "invalid") + for row in run.rows: + with self.subTest(tokens=row["tokens_per_rank"]): + self.assertIs(row["correctness"]["chain_last_output_passed"], False) + self.assertIs(row["correctness"]["post_chain_state_passed"], True) + self.assertIs(row["correctness"]["passed"], False) + # The magnitude rides along so the verdict can be judged, not just believed. + self.assertGreater(row["correctness"]["chain_last_output_error"], 0.0) + + def test_the_drained_oracles_still_red_the_case_on_their_own(self): + # The chained gate is an addition, not a replacement. + for phase in ("pre", "post"): + with self.subTest(phase=phase): + run = drive(fail_phases=(phase,)) + self.assertEqual(run.rc, 3) + self.assertEqual(run.doc["outcome"]["status"], "invalid") + for row in run.rows: + self.assertIs(row["correctness"]["passed"], False) + self.assertIs(row["correctness"]["post_chain_state_passed"], True) + self.assertIs(row["correctness"]["chain_last_output_passed"], True) + + def test_the_output_check_is_skipped_where_staging_is_hoisted(self): + # Under the hoist the chain captures one warm-up dispatch's staged stand-in and reuses + # it for every pair, so neither the chain's final combine nor the drained reference + # consumes an input matching its OWN dispatch. They are two differently mismatched + # pairs and nothing requires them to agree, so the question is not asked. + # Measured, h100/deepep-v2/EP8, identical but for the hoist: hoisted gives + # chain_last_output_error 31..93 (1000x-2966x tolerance), per-pair staging gives 0.0 at + # every rung, in both normal and low-latency mode. Gating on it under the hoist + # reddened every FP8 leg fleet-wide for a harness artifact. + class _Hoisting(_SweepBackend): + def __init__(self): + super().__init__() + self.precision, self.stage_device_work = "fp8", True + + backend = _Hoisting() + self.assertTrue(backend.stage_excluded_from_roundtrip) + run = drive(backend_factory=lambda: backend) + self.assertEqual(run.rc, 0) + self.assertEqual(run.output_checks, [], "the comparison must not run under the hoist") + for row in run.rows: + with self.subTest(tokens=row["tokens_per_rank"]): + # null, not False: the artifact says "not asked", never something a reader + # could mistake for a comparison that ran and failed. + self.assertIsNone(row["correctness"]["chain_last_output_passed"]) + self.assertIsNone(row["correctness"]["chain_last_output_error"]) + self.assertIs(row["correctness"]["passed"], True) + + def test_the_chained_error_is_folded_into_max_relative_error(self): + # Maxed in like the other two oracles', so a chained regime that is within tolerance + # but worse than the drained one stays visible. + run = drive(chain_error=0.25) + self.assertEqual(run.rc, 0) + for row in run.rows: + with self.subTest(tokens=row["tokens_per_rank"]): + self.assertAlmostEqual(row["correctness"]["max_relative_error"], 0.25) + + +class ChainedPublication(unittest.TestCase): + """What a free-running chain actually emits: values, origins, counts and placement.""" + + @classmethod + def setUpClass(cls): + cls.swept = drive() + + def test_every_published_chained_field(self): + # One walk over the emitted rows covering the whole chained family: the period and its + # origin, the per-op floors, and the three health scalars. Each carries its OWN origin + # and sample count because they are reduced differently -- median for the period, MIN + # for the floors, per-trial for the gap and drift -- and mixing those up is the defect + # this pins. + for row in self.swept.rows: + with self.subTest(tokens=row["tokens_per_rank"]): + period = row["components"]["pair_period"] + self.assertEqual(period["percentiles_us"]["p50"], PAIR_US) + self.assertEqual(period["origin"], "chained-median") + self.assertEqual(period["availability"], "measured") + self.assertEqual(period["sample_count"], KEPT_PER_TRIAL * CHAIN_TRIALS) + for op, expected in ( + ("dispatch", DISPATCH_FLOOR_US), ("combine", COMBINE_FLOOR_US), + ): + floor = row["chain_floor_us"][op] + self.assertEqual(floor["percentiles_us"]["p50"], expected) + self.assertEqual(floor["origin"], "chained-cross-rank-min") + health = row["chain_health"] + # One rank, so the ranks trivially agree; what matters is that the spread is + # emitted and component-shaped, since a wide spread disqualifies a period. + self.assertEqual(health["pair_spread_us"]["percentiles_us"]["p50"], 0.0) + self.assertEqual(set(health["pair_spread_us"]), set(UNAVAILABLE)) + # start-to-start median minus pair-window median: the per-pair cost OUTSIDE the + # published window, so instrumentation creeping back into the loop shows here. + gap = health["interpair_gap_us"] + self.assertEqual(gap["percentiles_us"]["p50"], GAP_US) + self.assertEqual(gap["availability"], "measured") + self.assertEqual(gap["sample_count"], CHAIN_TRIALS) + drift = health["settle_drift_us"] + self.assertEqual(drift["percentiles_us"]["p50"], 0.0) + self.assertEqual(drift["sample_count"], CHAIN_TRIALS) + # Additive: the fresh-entry family keeps its meaning alongside the chain. + self.assertEqual(row["components"]["roundtrip"]["origin"], "measured") + self.assertEqual(row["components"]["roundtrip"]["percentiles_us"]["p50"], 10.0) + self.assertEqual(row["components"]["dispatch"]["percentiles_us"]["p50"], 10.0) + + def test_the_doc_and_the_per_point_line_record_the_chain(self): + self.assertIs(self.swept.doc["implementation"]["chained_period"], True) + sampling = self.swept.doc["measurement"]["sampling"] + self.assertEqual(sampling["chain_iterations_per_trial"], CHAIN_ITERS) + self.assertEqual(sampling["chain_trials"], CHAIN_TRIALS) + self.assertEqual(sampling["chain_drop"], CHAIN_DROP) + self.assertIn("period=", self.swept.stdout) + self.assertNotIn("period=n/a", self.swept.stdout) + + +class _DriftingBackend(_SweepBackend): + """A chain whose late half runs DRIFT_US slower -- an unconverged (or down-clocking) run.""" + + def benchmark_chain(self, problem, warmup, iters, drop): + self.events.append(("chain", problem.T)) + kept = iters - drop + half = kept // 2 + pair = [PAIR_US] * half + [PAIR_US + DRIFT_US] * (kept - half) + return { + "pair": pair, + "start_to_start": [value + GAP_US for value in pair[:-1]], + "dispatch": [DISPATCH_FLOOR_US] * kept, + "combine": [COMBINE_FLOOR_US] * kept, + "combined": f"chained-{problem.T}", + } + + +class SettleDrift(unittest.TestCase): + """`chain_drop` assumes the chain settled by the time the kept iterations start, and nothing + else in the artifact could show that it hadn't -- so a drifting chain publishes its drift.""" + + def test_an_unconverged_chain_publishes_its_drift(self): + run = drive(backend_factory=_DriftingBackend) + # A health diagnostic, not a gate: the case stays green and the number says how much + # to distrust the period. + self.assertEqual(run.rc, 0) + for row in run.rows: + with self.subTest(tokens=row["tokens_per_rank"]): + drift = row["chain_health"]["settle_drift_us"] + self.assertEqual(drift["percentiles_us"]["p50"], DRIFT_US) + self.assertEqual(drift["sample_count"], CHAIN_TRIALS) + + +class _Vec: + """Just enough elementwise tensor for `_chain_output_matches`: a flat list of floats.""" + + def __init__(self, values): + self.values = [float(value) for value in values] + + @property + def shape(self): + return (len(self.values),) + + def numel(self): + return len(self.values) + + def float(self): + return _Vec(self.values) + + def abs(self): + return _Vec(abs(value) for value in self.values) + + def clamp_min(self, floor): + return _Vec(max(value, floor) for value in self.values) + + def __sub__(self, other): + return _Vec(a - b for a, b in zip(self.values, other.values)) + + def __truediv__(self, other): + return _Vec(a / b for a, b in zip(self.values, other.values)) + + def max(self): + return SimpleNamespace(item=lambda: max(self.values)) + + +class ChainOutputCheck(unittest.TestCase): + """The regime A/B judged the way the oracle judges combines: elementwise relative error + under a magnitude floor, because a combine kernel is not required to be order-deterministic + across invocations -- bit equality would red healthy backends.""" + + def test_the_verdict_and_the_magnitude_together(self): + # One table over the whole contract. The magnitude matters as much as the verdict: a + # bare bool cost two wrong diagnoses of the same FP8 failures, because it cannot + # separate a transport corruption from a tolerance too tight for an accumulator. + tol = ep_harness.COMBINE_REL_TOL + jitter = [value * (1.0 + tol / 2) for value in (1.0, -2.0)] + for label, chained, drained, ok, check in ( + ("identical", [1.0, -3.5, 0.25], [1.0, -3.5, 0.25], True, + lambda e: self.assertEqual(e, 0.0)), + # A rank that legitimately combined nothing under this routing. + ("empty", [], [], True, lambda e: self.assertEqual(e, 0.0)), + # Run-to-run jitter inside tolerance passes, and still reports its size -- a + # magnitude creeping toward the gate is the early warning a bool cannot give. + ("jitter", jitter, [1.0, -2.0], True, + lambda e: self.assertAlmostEqual(e, tol / 2)), + # The defect class this exists for lands orders of magnitude past tolerance. + ("corruption", [1.0, 2.0], [1.0, 4.0], False, + lambda e: self.assertGreater(e, 10 * tol)), + # No elementwise error is defined across shapes; infinity stops a cross-rank MAX + # reporting a small number for a structural mismatch. + ("shape", [1.0, 2.0], [1.0, 2.0, 3.0], False, + lambda e: self.assertEqual(e, float("inf"))), + ): + with self.subTest(label): + got, error = ep_harness._chain_output_matches(_Vec(chained), _Vec(drained)) + self.assertIs(got, ok) + check(error) + + def test_near_zero_elements_are_judged_against_the_magnitude_floor(self): + # Relative error against a denominator of 1e-6 would be huge; the floor keeps + # numerically-tiny elements from redding a healthy chain. + drained, chained = 1e-6, 1e-6 + 1e-4 + self.assertGreater( + abs(chained - drained) / abs(drained), ep_harness.COMBINE_REL_TOL + ) + self.assertTrue( + ep_harness._chain_output_matches(_Vec([chained]), _Vec([drained]))[0] + ) + + +# ---- from test_summarize_headline.py ---------------------------------------------- +ROUNDTRIP = {"p50": 100.0, "p90": 110.0, "p95": 115.0, "p99": 120.0} +PERIOD = {"p50": 60.0, "p90": 66.0, "p95": 69.0, "p99": 72.0} + + +def document(with_period): + components = { + "roundtrip": {"percentiles_us": dict(ROUNDTRIP)}, + } + if with_period: + components["pair_period"] = { + "percentiles_us": dict(PERIOD), "origin": "chained-median", + } + return { + "version": 1, + "outcome": {"status": "success"}, + "identity": { + "case_factors": { + "sku": "stub-sku", + "case": { + "backend": "stub", "suite": "ep-core", "routing": "uniform", + "mode": "low-latency", "phase": "decode", "ep": 8, + "precision": "bf16", + }, + }, + }, + "topology": {"gpus_per_node": 8, "scale_up_domain": 8, "nodes": 1}, + "measurement": { + "rows": [{ + "tokens_per_rank": 64, + "components": components, + "logical_copies": {"wire": "per-assignment"}, + "cross_rank_min_us": {"roundtrip": {"percentiles_us": {"p50": 90.0}}}, + "cross_rank_spread_us": {"percentiles_us": {"p50": 5.0}}, + }], + }, + } + + +class Headline(unittest.TestCase): + def test_the_headline_is_the_pair_period_when_a_row_carries_one(self): + tokens, p50, p99, _, _, carries = summarize._headline(document(with_period=True)) + self.assertEqual((tokens, p50, p99), (64, PERIOD["p50"], PERIOD["p99"])) + self.assertTrue(carries) + self.assertIn("chained pair period", summarize.render([document(with_period=True)])) + + def test_a_row_without_a_period_falls_back_to_the_roundtrip(self): + _, p50, p99, _, _, carries = summarize._headline(document(with_period=False)) + self.assertEqual((p50, p99), (ROUNDTRIP["p50"], ROUNDTRIP["p99"])) + self.assertFalse(carries) + self.assertIn( + "no row here carries a chained pair period", + summarize.render([document(with_period=False)]), + ) + + +class RunnerFilterIsContentBased(unittest.TestCase): + # --runner filters on the SKU the row declares, not on the result filename: results are + # named by case_id, which joins its factors with "-", so any filename-prefix test would + # match nothing and render an empty table rather than failing. + def _directory(self, names): + directory = tempfile.mkdtemp() + for name, sku in names: + record = document(with_period=True) + # The shared fixture omits record_type (nothing else loads from disk); + # load_results discriminates on it, so a written record must carry it. + record["record_type"] = summarize.CASE_RECORD_TYPE + record["identity"]["case_factors"]["sku"] = sku + (Path(directory) / name).write_text(json.dumps(record)) + return directory + + def test_case_id_named_files_still_match_their_runner(self): + directory = self._directory([ + ("stub-sku-stub-deepseek-v3-low-latency-decode-ep8-uniform-bf16_TS-c000.json", + "stub-sku"), + ("other-sku-stub-deepseek-v3-low-latency-decode-ep8-uniform-bf16_TS-c000.json", + "other-sku"), + ]) + kept = summarize.load_results(directory, "stub-sku", None) + self.assertEqual(len(kept), 1) + self.assertEqual(kept[0]["identity"]["case_factors"]["sku"], "stub-sku") + + def test_a_foreign_sku_is_excluded_even_when_the_filename_would_match(self): + # The filename claims one SKU, the row declares another; the row wins. + directory = self._directory([("stub-sku_anything_TS-c000.json", "other-sku")]) + self.assertEqual(summarize.load_results(directory, "stub-sku", None), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/experimental/CollectiveX/tests/test_ep_backend.py b/experimental/CollectiveX/tests/test_ep_backend.py deleted file mode 100644 index a9c38f3c9..000000000 --- a/experimental/CollectiveX/tests/test_ep_backend.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -"""Small torch-free smoke tests for the shared EP backend lifecycle.""" -from __future__ import annotations - -import sys -import types -import unittest -from pathlib import Path -from unittest import mock - -ROOT = Path(__file__).resolve().parents[1] -sys.path[:0] = [str(ROOT), str(ROOT / "bench")] - -import ep_backend # noqa: E402 -from ep_backend import EPBackend, RankInputs # noqa: E402 - - -def args(**updates): - values = dict( - experts=8, phase="decode", tokens_ladder="", routing="uniform", seed=0, - hidden=16, topk=2, mode="normal", precision="bf16", - ) - values.update(updates) - return types.SimpleNamespace(**values) - - -class FakeBackend(EPBackend): - name = "fake" - - def __init__(self, options, *, cap=None, world_size=1): - super().__init__(options, 0, world_size, 0, "cpu") - self.cap = cap - self.calls: list[str] = [] - - def create_buffer(self, spec): - return None - - def dispatch(self, problem): - self.calls.append("dispatch") - return object() - - def stage(self, problem, handle): - self.calls.append("stage") - - def combine(self, problem, handle): - self.calls.append("combine") - - def recv_tokens(self, handle): - return 0 - - def inspect_dispatch(self, problem, handle): - return None - - def combine_transformed(self, problem, handle, transformed): - return None - - def buffer_cap(self, options): - return self.cap - - def _build_rank_inputs(self, options, tokens): - return RankInputs( - tokens_per_rank=tokens, topk_idx=None, topk_weights=None, - activations=None, - ) - - -class BackendTests(unittest.TestCase): - def test_input_plan_sizes_for_the_measured_ladder(self): - backend = FakeBackend(args(tokens_ladder="8 16"), world_size=2) - spec = backend.make_inputs(backend.args) - self.assertTrue(spec.ok) - self.assertEqual(spec.ladder, [8, 16]) - self.assertEqual(spec.max_tokens_per_rank, 16) - self.assertEqual((spec.ep_size, spec.experts_per_rank), (2, 4)) - self.assertEqual(sorted(spec.points), [8, 16]) - - def test_invalid_or_fully_clamped_ladder_fails_before_execution(self): - for backend, message in ( - (FakeBackend(args(tokens_ladder="0")), "empty token ladder"), - (FakeBackend(args(tokens_ladder="128"), cap=64), "cap=64"), - ): - with self.subTest(message=message): - spec = backend.make_inputs(backend.args) - self.assertEqual(spec.rc, 2) - self.assertIn(message, spec.message) - - def test_timed_components_follow_backend_contract(self): - backend = FakeBackend(args()) - self.assertEqual(backend.timed_components(), ["roundtrip", "dispatch", "combine"]) - backend.stage_device_work = True - self.assertEqual( - backend.timed_components(), ["roundtrip", "dispatch", "combine", "stage"] - ) - backend.roundtrip_only = True - self.assertEqual(backend.timed_components(), ["roundtrip"]) - - def test_dispatch_cleanup_is_outside_timed_call(self): - backend = FakeBackend(args()) - backend.dispatch_needs_combine_cleanup = True - captured = {} - - def fake_time(_torch, operation, _warmup, _iters, **kwargs): - handle = operation() - kwargs["post"](handle) - captured.update(kwargs) - return [1.0] - - with mock.patch.dict(sys.modules, {"torch": types.SimpleNamespace()}), mock.patch.object( - ep_backend, "time_us", side_effect=fake_time - ): - backend.benchmark_dispatch(object(), 0, 1) - self.assertIn("post", captured) - self.assertEqual(backend.calls, ["dispatch", "stage", "combine"]) - - def test_stage_cleanup_matches_the_dispatch_contract(self): - # MoRI-shaped backends (dispatch_needs_combine_cleanup) must not leak an - # un-combined dispatch out of an isolated-stage iteration. - for needs_cleanup, calls in ( - (True, ["dispatch", "stage", "combine"]), (False, ["dispatch", "stage"]), - ): - backend = FakeBackend(args()) - backend.dispatch_needs_combine_cleanup = needs_cleanup - - def fake_time(_torch, operation, _warmup, _iters, **kwargs): - result = operation(kwargs["pre"]()) - if kwargs["post"] is not None: - kwargs["post"](result) - return [1.0] - - with mock.patch.dict(sys.modules, {"torch": types.SimpleNamespace()}), mock.patch.object( - ep_backend, "time_us", side_effect=fake_time - ): - backend.benchmark_stage(object(), 0, 1) - with self.subTest(needs_cleanup=needs_cleanup): - self.assertEqual(backend.calls, calls) - - def test_mode_is_fail_closed(self): - with self.assertRaises(ValueError): - FakeBackend(args(mode="unsupported")) - - def test_low_latency_mode_accepted_only_when_declared(self): - # The base backend is normal-only, so it must reject low-latency; an adapter that - # declares it in SUPPORTED_MODES is accepted and can carry the weighted-kernel - # combine semantics the low-latency oracle path keys on. - with self.assertRaises(ValueError): - FakeBackend(args(mode="low-latency")) - - class LowLatencyBackend(FakeBackend): - SUPPORTED_MODES = ("normal", "low-latency") - - backend = LowLatencyBackend(args(mode="low-latency")) - backend.combine_weight_semantics = "weighted-kernel-sum" - self.assertEqual(backend.mode, "low-latency") - self.assertEqual(backend.combine_weight_semantics, "weighted-kernel-sum") - - def test_precision_is_fail_closed(self): - # The base SUPPORTED_PRECISIONS is BF16-only; an adapter that has not opted - # into a precision must reject it rather than silently run the wrong codec. - with self.assertRaises(ValueError): - FakeBackend(args(precision="fp8")) - - def test_base_dispatch_encoding_is_identity(self): - # BF16 default: semantic_payload is identity and make_problem attaches no - # oracle_x, so the combine oracle falls back to problem.x (unchanged behavior). - backend = FakeBackend(args()) - payload = object() - self.assertIs(backend.semantic_payload(payload), payload) - self.assertEqual(backend._encode_dispatch(payload), (payload, None)) - self.assertEqual(backend.dispatch_dtype, "bf16") - self.assertEqual(backend.combine_dtype, "bf16") - - -if __name__ == "__main__": - unittest.main() diff --git a/experimental/CollectiveX/tests/test_ep_nccl_handle.py b/experimental/CollectiveX/tests/test_ep_nccl_handle.py deleted file mode 100644 index 943625ee7..000000000 --- a/experimental/CollectiveX/tests/test_ep_nccl_handle.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -"""Torch-free tests for the nccl-ep single-handle contract. - -The invariant under test: ONE handle per group, rebound per problem shape. Two handles built -from the same group config resolve to the same LL parity signal slots while advancing their -parity independently, which corrupts signalling (NVIDIA/nccl#2303) -- so a regression that -reintroduces per-shape handles must fail loudly here rather than on a cluster. - -torch and nccl are stubbed so this runs without the benchmark image. -""" -from __future__ import annotations - -import sys -import types -import unittest -from pathlib import Path -from unittest import mock - -ROOT = Path(__file__).resolve().parents[1] - - -def _stub_modules(): - """Fake torch / nccl modules so `import ep_nccl` succeeds without the benchmark image.""" - torch = types.ModuleType("torch") - torch.bfloat16 = "bfloat16" - torch.int32 = "int32" - torch.empty = lambda *a, **k: types.SimpleNamespace(shape=a[0] if a else ()) - torch.zeros = lambda *a, **k: types.SimpleNamespace(item=lambda: 7) - torch.cuda = types.SimpleNamespace(synchronize=lambda: None) - dist = types.ModuleType("torch.distributed") - torch.distributed = dist - - ep = types.ModuleType("nccl.ep") - for name in ( - "Algorithm", "CombineConfig", "CombineInputs", "CombineOutputs", "DispatchConfig", - "DispatchInputs", "DispatchOutputs", "GroupConfig", "HandleConfig", "Layout", - "LayoutInfo", "Tensor", - ): - setattr(ep, name, type(name, (), {"__init__": lambda self, *a, **k: None})) - ep.Algorithm = types.SimpleNamespace(LOW_LATENCY="LL", HIGH_THROUGHPUT="HT") - ep.Layout = types.SimpleNamespace(EXPERT_MAJOR="EM", FLAT="FLAT") - core = types.ModuleType("nccl.core") - pkg = types.ModuleType("nccl") - pkg.ep, pkg.core = ep, core - return { - "torch": torch, "torch.distributed": dist, - "nccl": pkg, "nccl.ep": ep, "nccl.core": core, - } - - -sys.path[:0] = [str(ROOT), str(ROOT / "bench")] - -# Import ep_nccl against the stubs, then withdraw them: leaving a fake torch in sys.modules -# makes the genuinely torch-dependent modules in this process (test_runtime, test_ll_oracle) -# error instead of skipping. Dropping ep_nccl too keeps the stub-built module private to us. -with mock.patch.dict(sys.modules, _stub_modules()): - import ep_nccl # noqa: E402 - - sys.modules.pop("ep_nccl", None) - - -class FakeHandle: - """Records every rebind so the tests can assert on the collective call pattern.""" - - def __init__(self): - self.updates = [] - self.destroyed = False - - def update(self, topk_idx, *, layout_info=None, stream=None): - self.updates.append((topk_idx, layout_info)) - - def destroy(self): - self.destroyed = True - - -class FakeGroup: - def __init__(self): - self.created = 0 - self.handle = FakeHandle() - - def create_handle(self, layout, topk_idx, *, layout_info=None, config=None, stream=None): - self.created += 1 - return self.handle - - -def backend(ll=True): - """An NCCLEPBackend with just the fields _ensure_handle touches (no __init__, no GPU).""" - b = object.__new__(ep_nccl.NCCLEPBackend) - b._ll = ll - b._layout = "EM" if ll else "FLAT" - b._handle = None - b._bound = None - b._ep_group = FakeGroup() - b.device = "cuda:0" - b.num_local_experts = 4 - b.args = types.SimpleNamespace(hidden=16) - b._t = lambda x: x - b._stream = lambda: 0 - return b - - -def problem(T): - return types.SimpleNamespace( - T=T, dispatch_x=f"x{T}", topk_idx=f"idx{T}", topk_weights=f"w{T}" - ) - - -class TestSingleHandle(unittest.TestCase): - def test_one_handle_across_many_shapes(self): - """Nine ladder rungs must still produce exactly one create_handle.""" - b = backend() - for T in (1, 2, 4, 8, 16, 32, 64, 128, 256): - b._ensure_handle(problem(T)) - self.assertEqual(b._ep_group.created, 1) - - def test_shape_change_rebinds_and_repeat_does_not(self): - """update() on a shape switch; no collective when the bound shape is re-entered.""" - b = backend() - pa, pb = problem(1), problem(2) - b._ensure_handle(pa) - self.assertEqual(len(b._ep_group.handle.updates), 0) # first bind is the create - - b._ensure_handle(pb) - self.assertEqual(len(b._ep_group.handle.updates), 1) - - # Re-entering the bound problem repeatedly -- the timed loop's steady state -- must not - # enter a collective, otherwise every iteration gains a rank-synchronising step. - for _ in range(8): - b._ensure_handle(pb) - self.assertEqual(len(b._ep_group.handle.updates), 1) - - # Returning to an earlier shape rebinds again (its cached namespace is reused). - b._ensure_handle(pa) - self.assertEqual(len(b._ep_group.handle.updates), 2) - - def test_every_problem_shares_the_one_handle(self): - b = backend() - handles = {id(b._ensure_handle(problem(T)).handle) for T in (1, 2, 4)} - self.assertEqual(len(handles), 1) - self.assertIs(b._ensure_handle(problem(1)).handle, b._handle) - - def test_ll_never_passes_layout_info_on_rebind(self): - """The API forbids layout_info on create/update in LL mode.""" - b = backend(ll=True) - b._ensure_handle(problem(1)) - b._ensure_handle(problem(2)) - self.assertEqual([info for _, info in b._ep_group.handle.updates], [None]) - - def test_ht_rebind_carries_that_problems_counters(self): - """HT re-runs the metadata exchange into the rebound problem's own counter tensors.""" - b = backend(ll=False) - ha = b._ensure_handle(problem(1)) - hb = b._ensure_handle(problem(2)) - self.assertEqual(len(b._ep_group.handle.updates), 1) - self.assertIs(b._ep_group.handle.updates[0][1], hb.layout_info) - self.assertIsNot(ha.layout_info, hb.layout_info) - self.assertEqual(hb.count, 7) # re-read after the exchange - - def test_destroy_releases_the_handle_once(self): - b = backend() - b._ensure_handle(problem(1)) - handle = b._handle - b._destroy_handles() - self.assertTrue(handle.destroyed) - self.assertIsNone(b._handle) - self.assertIsNone(b._bound) - b._destroy_handles() # idempotent - - -if __name__ == "__main__": - unittest.main() diff --git a/experimental/CollectiveX/tests/test_matrix.py b/experimental/CollectiveX/tests/test_matrix.py index f89035b79..730556dab 100644 --- a/experimental/CollectiveX/tests/test_matrix.py +++ b/experimental/CollectiveX/tests/test_matrix.py @@ -7,37 +7,51 @@ import tempfile import unittest from pathlib import Path +from unittest import mock ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "runtime")) + import sweep_matrix # noqa: E402 +import config # noqa: E402 def matrix(**options): return sweep_matrix.resolve_matrix(**options) -class MatrixTests(unittest.TestCase): - def test_shard_extraction_is_deterministic_and_preserves_cases(self): - document = matrix(backend="deepep-v2", only_sku="h200-dgxc") - cell = document["include"][0] - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - source = root / "matrix.json" - source.write_text(json.dumps(document, sort_keys=True)) - outputs = [ - sweep_matrix.extract_shard( - source, cell["id"], root / f"shard-{index}.json", - ) - for index in range(2) - ] - self.assertEqual(outputs[0], outputs[1]) - self.assertEqual(outputs[0]["cases"], cell["cases"]) +def cells(document, project=("sku", "ep"), **filters): + """Projected set of the requested cases matching `filters`. + + The rollout tests all ask the same question -- which (sku, ep) pairs a backend offers, and + under which disposition -- so they share one query instead of restating the comprehension. + `project` names the fields to return ("sku" comes from the item, everything else from its + case); a single field projects to scalars rather than 1-tuples. + """ + fields = (project,) if isinstance(project, str) else project + selected = set() + for item in document["requested_cases"]: + case = item["case"] + if any( + (item["disposition"] if key == "disposition" else case[key]) != value + for key, value in filters.items() + ): + continue + row = tuple(item["sku"] if f == "sku" else case[f] for f in fields) + selected.add(row[0] if isinstance(project, str) else row) + return selected + +class MatrixTests(unittest.TestCase): def test_sku_and_ep_filters_only_remove_cases(self): + # Subtractive with ONE deliberate exception: naming an off-path precision explicitly + # opts its rows back in (see OFF_PATH_PRECISIONS), so the fp8 subset is compared + # against a baseline that also names fp8 rather than against the default matrix. full = matrix(backend="all") - for options, keep in ( + full_with_off_path = matrix(backend="all", precisions="bf16,fp8") + for case in ( ({"exclude_skus": "b300"}, lambda item: item["sku"] != "b300"), ({"ep_sizes": "8"}, lambda item: item["case"]["ep"] == 8), # A precision subset removes only the runnable cases of the other @@ -45,7 +59,7 @@ def test_sku_and_ep_filters_only_remove_cases(self): ({"precisions": "bf16"}, lambda item: item["case"]["precision"] == "bf16"), ({"precisions": "fp8"}, lambda item: item["case"]["precision"] == "fp8" - or item["disposition"] == "unsupported"), + or item["disposition"] == "unsupported", "off_path"), # A mode subset removes only the runnable cases of the other mode; the # ep-unsupported placeholder is normal-mode and mode-filter-independent, so it # survives both selections (mirrors the precision rows above). @@ -54,9 +68,12 @@ def test_sku_and_ep_filters_only_remove_cases(self): lambda item: item["case"]["mode"] == "low-latency" or item["disposition"] == "unsupported"), ): + options, keep = case[0], case[1] partial = matrix(backend="all", **options) + baseline = full_with_off_path if len(case) > 2 else full expected = { - item["case"]["case_id"]: item for item in full["requested_cases"] if keep(item) + item["case"]["case_id"]: item + for item in baseline["requested_cases"] if keep(item) } actual = {item["case"]["case_id"]: item for item in partial["requested_cases"]} self.assertEqual(actual, expected) @@ -78,35 +95,6 @@ def test_only_real_platform_cells_are_unsupported(self): for item in document["requested_cases"]: self.assertIn(item["case"]["backend"], sweep_matrix.PLATFORMS[item["sku"]]["backends"]) - def test_runnable_cases_fan_out_over_backend_precisions(self): - document = matrix(backend="all") - runnable = [ - item for item in document["requested_cases"] - if item["disposition"] == "runnable" - ] - # Every runnable case carries a precision its backend supports, and each - # (sku, backend, ep, phase) cell is realized once per supported precision. - by_cell: dict[tuple, set[str]] = {} - for item in runnable: - case = item["case"] - self.assertIn( - case["precision"], sweep_matrix.BACKEND_PRECISIONS[case["backend"]] - ) - cell = (item["sku"], case["backend"], case["ep"], case["phase"]) - by_cell.setdefault(cell, set()).add(case["precision"]) - for cell, precisions in by_cell.items(): - expected = { - precision for precision in sweep_matrix.SWEEP["precisions"] - if precision in sweep_matrix.BACKEND_PRECISIONS[cell[1]] - } - self.assertEqual(precisions, expected, cell) - # Every backend that lists FP8 (deepep-v2, mori, uccl-ep) realizes BF16 and FP8; - # nccl-ep is BF16-only, so the cross-cell union of realized precisions stays {bf16, fp8}. - self.assertEqual( - {precision for precisions in by_cell.values() for precision in precisions}, - {"bf16", "fp8"}, - ) - def test_case_ids_are_unique_across_the_matrix(self): # precision is part of case_id, so a cell's bf16 and fp8 attempts are distinct # identities. Without precision in the id the two would collide; assert the full @@ -118,39 +106,6 @@ def test_case_ids_are_unique_across_the_matrix(self): for item in document["requested_cases"]: self.assertTrue(item["case"]["case_id"].endswith(item["case"]["precision"])) - def test_low_latency_is_decode_only_and_capability_gated(self): - # Low-latency cases are additive: they appear only for (sku, backend, ep) cells - # listed in the platform registry's ll_backends map, only in the decode phase, and - # never as unsupported placeholders. Normal-mode cases are unchanged by their - # presence. - document = matrix(backend="all") - ll = [ - item for item in document["requested_cases"] - if item["case"]["mode"] == "low-latency" - ] - self.assertTrue(ll, "expected at least one low-latency cell in the registry") - for item in ll: - case = item["case"] - self.assertEqual(item["disposition"], "runnable") - self.assertEqual(case["phase"], "decode") - self.assertIn("low-latency", sweep_matrix.SWEEP["modes"]) - ll_backends = sweep_matrix.PLATFORMS[item["sku"]].get("ll_backends", {}) - self.assertIn(case["ep"], ll_backends.get(case["backend"], [])) - self.assertIn("-low-latency-", case["case_id"]) - # Every low-latency cell realizes exactly its backend's supported precisions. - by_cell: dict[tuple, set[str]] = {} - for item in ll: - case = item["case"] - cell = (item["sku"], case["backend"], case["ep"]) - by_cell.setdefault(cell, set()).add(case["precision"]) - for cell, precisions in by_cell.items(): - self.assertEqual( - precisions, - {p for p in sweep_matrix.SWEEP["precisions"] - if p in sweep_matrix.BACKEND_PRECISIONS[cell[1]]}, - cell, - ) - def test_ll_backends_is_a_well_formed_subset_of_backends(self): # A cell can only run low-latency where it can run at all: every ll_backends # entry names a real backend of that SKU and a subset of its normal EP degrees. @@ -162,62 +117,6 @@ def test_ll_backends_is_a_well_formed_subset_of_backends(self): self.assertTrue(degrees) self.assertLessEqual(set(degrees), set(platform["backends"][backend])) - def test_ep_degrees_pin_the_two_backends_without_a_rollout_test(self): - # deepep-v2 and mori have no rollout-shape test of their own, so the two facts most - # likely to drift silently are pinned here: deepep-v2 is the only backend at EP16 - # everywhere, and mori is EP8-only (its EP16 InterNodeV1 combine corrupts, mori#475). - # Without this, adding 16 to backends.mori passes CI. - for sku, platform in sweep_matrix.PLATFORMS.items(): - degrees = platform["backends"] - with self.subTest(sku=sku): - if "mori" in degrees: - self.assertEqual(degrees["mori"], [8]) - if "deepep-v2" in degrees: - self.assertEqual(degrees["deepep-v2"], [8, 16]) - - def test_uccl_ep_rollout_shape(self): - # UCCL-EP's rollout, locked here: EP8 runnable on exactly the six supported SKUs, and - # EP16 an unsupported coverage row on every one of them. uccl-ep is EP8-only: the - # -tw pair has no cross-node fabric, and on the fabric SKUs cross-node EP16 is - # functional but its CPU-proxy throughput overruns the standardized per-case - # wall-clock budget (the internode Config fix landed; EP16 stays scoped out of the - # sweep, mirroring the mori EP16 re-wall). No rows at all on b300/gb200/gb300, where - # the backend is not offered. LL (decode) on every NVIDIA supported SKU at EP8. - document = matrix(backend="all") - runnable = { - (item["sku"], item["case"]["ep"]) - for item in document["requested_cases"] - if item["case"]["backend"] == "uccl-ep" and item["disposition"] == "runnable" - } - unsupported = { - (item["sku"], item["case"]["ep"]) - for item in document["requested_cases"] - if item["case"]["backend"] == "uccl-ep" and item["disposition"] == "unsupported" - } - supported_skus = { - "h100-dgxc", "h200-dgxc", "b200-dgxc", "mi355x", "mi325x-tw", "mi300x-tw", - } - # EP8 runnable on all six; nothing runnable at EP16. - self.assertEqual({sku for sku, _ in runnable}, supported_skus) - self.assertEqual({sku for sku, ep in runnable if ep == 8}, supported_skus) - self.assertEqual({sku for sku, ep in runnable if ep == 16}, set()) - # EP16 is an honest unsupported coverage row on every supported SKU. - self.assertEqual(unsupported, {(sku, 16) for sku in supported_skus}) - offered = {sku for sku, _ in runnable | unsupported} - for absent in ("b300", "gb200", "gb300"): - self.assertNotIn(absent, offered) - # uccl-ep low-latency is enabled only on NVIDIA; the AMD SKUs keep normal mode but drop - # LL: upstream raised kNumMaxTopK 9 -> 16 six days before our pin, and the host assert - # kNumMaxTopK + 1 <= num_warp_groups * num_warps_per_group cannot hold on AMD, whose - # kNumMaxWarpGroups is 16 — a dated regression, not a CU-count limit. - ll_skus = { - item["sku"] - for item in document["requested_cases"] - if item["case"]["backend"] == "uccl-ep" - and item["case"]["mode"] == "low-latency" - } - self.assertEqual(ll_skus, {"h100-dgxc", "h200-dgxc", "b200-dgxc"}) - def test_flashinfer_ep_rollout_shape(self): # FlashInfer one-sided is the transport a GB deployment actually runs: vLLM picks # `flashinfer_nvlink_one_sided` on NVLink and `deepep_v2` on RDMA, so it belongs on the @@ -231,7 +130,9 @@ def test_flashinfer_ep_rollout_shape(self): # a separate decode kernel, so an ll_backends cell would re-measure the same kernel # under a mode that promises a different one. Decode is still covered by the decode # phase of normal mode. - # BF16 only this pass (FP8 dispatch needs the scale payload plumbed and oracle-validated). + # BF16 only in the DEFAULT matrix: the FP8 dispatch is implemented and oracle-validated, + # but no engine can select it on this transport, so OFF_PATH_PRECISIONS keeps it out + # unless `--precisions` names fp8 explicitly. document = matrix(backend="all") cases = [ item for item in document["requested_cases"] @@ -242,76 +143,21 @@ def test_flashinfer_ep_rollout_shape(self): for item in cases if item["disposition"] == "runnable" } self.assertEqual(runnable, {(sku, ep) for sku in ("gb200", "gb300") for ep in (8, 16)}) + # FP8 is dispatch-side only: scales ride as a fourth payload (kMaxPayloads is exactly 4) + # and combine stays BF16, so none of the 0.6.16+ combine-quant API is needed. It is + # realizable but off-path, so the DEFAULT matrix carries BF16 alone and naming the + # precision brings it back for transport comparison. self.assertEqual({item["case"]["precision"] for item in cases}, {"bf16"}) + opted_in = cells( + matrix(backend="flashinfer-ep", precisions="fp8"), "precision", + disposition="runnable", + ) + self.assertEqual(opted_in, {"fp8"}) # Normal mode only — no low-latency cell on any SKU. self.assertEqual({item["case"]["mode"] for item in cases}, {"normal"}) for platform in sweep_matrix.PLATFORMS.values(): self.assertNotIn("flashinfer-ep", platform.get("ll_backends", {})) - def test_nccl_ep_rollout_shape(self): - # NCCL-EP's rollout, locked to the on-metal verdict (2026-07-22, all via the real launcher): - # * RDMA scale-out SKUs (h100/h200/b200/b300): EP8 runnable, EP16 an UNSUPPORTED coverage - # row. EP16 cross-node rides NCCL's kernel-initiated GDAKI GIN, which faults identically - # on RoCE (h100/b200/b300) and InfiniBand (h200) — a reproducible NCCL-EP v0.1.0 internode - # limitation, so EP16 is scoped out like uccl-ep's. - # * GB NVL72 SKUs (gb200/gb300, MNNVL, gb-nv launcher, 4 GPU/node): EP8 AND EP16 runnable — - # both stay inside the 72-GPU scale-up domain (world <= scale_up_domain => LSA, no GIN), - # so EP16 works over MNNVL where the RDMA-GIN path walls. - # * AMD SKUs: no rows (NCCL EP is NVIDIA-only; AMD runs mori). - # * LOW-LATENCY: EP8 on all six NVIDIA SKUs. These rows were dropped while every LL leg - # wedged, on the reading that only a fixed wheel could restore them. That reading was - # wrong about the cause: the wedge needed TWO LL handles live on one group. `buffer_idx` - # is per-handle but its buffers are offsets into the per-group rdma_buffer, so same-config - # handles alias one another's parity count/flag slots. The adapter now binds a single - # handle and rebinds it per shape, which removes the aliasing without a wheel bump — - # proven on a stock wheel by ladder [1] (one handle, clean) vs [1, 2] (two handles, - # 64 dispatch + 6 combine receive timeouts). LL is decode-only and EP8-only: GB stays at - # EP8 here even though normal mode runs EP16, because LL adds no EP16 row on any SKU. - # BF16 only — no FP8 case (NCCL EP FP8 unsupported this release). - document = matrix(backend="all") - runnable = { - (item["sku"], item["case"]["ep"]) - for item in document["requested_cases"] - if item["case"]["backend"] == "nccl-ep" and item["disposition"] == "runnable" - } - unsupported = { - (item["sku"], item["case"]["ep"]) - for item in document["requested_cases"] - if item["case"]["backend"] == "nccl-ep" and item["disposition"] == "unsupported" - } - rdma_skus = {"h100-dgxc", "h200-dgxc", "b200-dgxc", "b300"} - gb_skus = {"gb200", "gb300"} - # RDMA SKUs: EP8 runnable + EP16 unsupported. GB SKUs: EP8 and EP16 both runnable. - self.assertEqual( - runnable, - {(sku, 8) for sku in rdma_skus} | {(sku, ep) for sku in gb_skus for ep in (8, 16)}, - ) - self.assertEqual(unsupported, {(sku, 16) for sku in rdma_skus}) - # Offered on the six NVIDIA SKUs; never on AMD. - offered = {sku for sku, _ in runnable | unsupported} - self.assertEqual(offered, rdma_skus | gb_skus) - for absent in ("mi355x", "mi325x-tw", "mi300x-tw"): - self.assertNotIn(absent, offered) - # Every nccl-ep case is BF16 (FP8 unsupported this release). - self.assertEqual( - { - item["case"]["precision"] - for item in document["requested_cases"] - if item["case"]["backend"] == "nccl-ep" - }, - {"bf16"}, - ) - # Low-latency: EP8 on every NVIDIA SKU, and EP8 only — a stray EP16 LL row would dispatch a - # shape the mode does not define. - ll = { - (item["sku"], item["case"]["ep"]) - for item in document["requested_cases"] - if item["case"]["backend"] == "nccl-ep" - and item["case"]["mode"] == "low-latency" - and item["disposition"] == "runnable" - } - self.assertEqual(ll, {(sku, 8) for sku in rdma_skus | gb_skus}) - def test_invalid_filters_fail_closed(self): for options in ( {"exclude_skus": "unknown"}, @@ -326,6 +172,26 @@ def test_invalid_filters_fail_closed(self): sweep_matrix.resolve_matrix(**options) +class UndeclaredPrecisionsFailClosed(unittest.TestCase): + # A backend in platform_config but missing from BACKEND_PRECISIONS must stop the matrix + # rather than resolve to bf16-only: that yields a MISSING case, not a mislabelled one, and + # run_sweep's non-bf16-dispatch guard can only catch cases that ran. + def test_a_backend_without_declared_precisions_stops_the_matrix(self): + pruned = { + name: value for name, value in sweep_matrix.BACKEND_PRECISIONS.items() + if name != "deepep-v2" + } + with mock.patch.object(sweep_matrix, "BACKEND_PRECISIONS", pruned): + with self.assertRaises(SystemExit) as caught: + sweep_matrix.resolve_matrix() + self.assertIn("deepep-v2", str(caught.exception)) + self.assertIn("BACKEND_PRECISIONS", str(caught.exception)) + + def test_every_scheduled_backend_declares_its_precisions(self): + for backend in sweep_matrix.SWEEP_BACKENDS: + self.assertIn(backend, sweep_matrix.BACKEND_PRECISIONS, backend) + + class BackendMaturityTests(unittest.TestCase): """The registry map and each adapter's `maturity` are two copies of one fact, read by different consumers, so they can drift silently: pin coverage, vocabulary and agreement. diff --git a/experimental/CollectiveX/tests/test_ll_oracle.py b/experimental/CollectiveX/tests/test_measurement.py similarity index 54% rename from experimental/CollectiveX/tests/test_ll_oracle.py rename to experimental/CollectiveX/tests/test_measurement.py index 08d2ea37a..f3daeadc2 100644 --- a/experimental/CollectiveX/tests/test_ll_oracle.py +++ b/experimental/CollectiveX/tests/test_measurement.py @@ -1,15 +1,5 @@ #!/usr/bin/env python3 -"""End-to-end check of the low-latency per-slot correctness oracle on CPU. - -The real low-latency kernels only run on GPU, but the oracle's plumbing — -per-(source, expert) slot normalization, the delivered-assignment multiset check, -per-expert counts, and the gate-weighted combine comparison — is platform-independent -Python. This drives `_run_ll_expert_oracle` against a single-rank CPU fake that -implements CORRECT low-latency semantics (deliver one slot per local (token, expert) -assignment; combine = source-side gate-weighted sum), so a structural bug in the oracle -or a divergence between its expected-combine model and true low-latency behavior fails -here rather than silently reding every GPU leg. -""" +"""Measurement-model tests: the low-latency correctness oracle, and the bandwidth math.""" from __future__ import annotations import sys @@ -20,7 +10,12 @@ ROOT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(ROOT), str(ROOT / "bench")] +sys.path[:0] = [str(Path(__file__).resolve().parents[1])] +import bandwidth # noqa: E402 + + +# ---- from test_ll_oracle.py ------------------------------------------------------- try: import torch as _torch except Exception: # torch is absent in the plain CPU test image; runs on GPU CI @@ -35,6 +30,7 @@ class _FakeLLBackend: per-expert order and combines by the source-side gate-weighted sum the kernel does.""" name = "fake-ll" + receive_layout = "token-expert" combine_weight_semantics = "weighted-kernel-sum" def __init__(self, experts_per_rank: int, seed: int): @@ -97,23 +93,6 @@ def _problem(self, T: int, hidden: int, topk: int, experts: int, seed: int): ) return problem, idx_g, w_g - def test_correct_ll_backend_passes_every_oracle_check(self): - import routing - - torch = _torch - T, hidden, topk, experts = 8, 128, 4, 16 # ep_size 1 -> experts_per_rank == experts - problem, idx_g, w_g = self._problem(T, hidden, topk, experts, seed=67) - backend = _FakeLLBackend(experts_per_rank=experts, seed=67) - with mock.patch.object(torch.cuda, "synchronize", lambda *a, **k: None): - report = ep_harness._run_ll_expert_oracle( - torch, routing, backend, problem, idx_g, w_g, - rank=0, experts_per_rank=experts, scale_up_domain=1, seed=67, - ) - self.assertTrue(report["passed"], report["checks"]) - for name, ok in report["checks"].items(): - self.assertTrue(ok, f"check {name} failed: {report}") - self.assertLess(report["max_elementwise_relative_error"], ep_harness.COMBINE_REL_TOL) - def test_corrupted_combine_trips_the_gate(self): import routing @@ -133,5 +112,72 @@ def test_corrupted_combine_trips_the_gate(self): self.assertFalse(report["checks"]["combine_values"]) +# ---- from test_bandwidth.py ------------------------------------------------------- +COMPONENTS = bandwidth.COMPONENTS + + +def _row(tokens, nbytes, latency, passed=True): + """A measurement row. `latency` is a scalar, or a per-component dict whose None marks + that component unavailable.""" + lat = latency if isinstance(latency, dict) else dict.fromkeys(COMPONENTS, latency) + return { + "tokens_per_rank": tokens, + "components": {c: {"percentiles_us": None if lat[c] is None else { + "p50": lat[c], "p90": lat[c], "p95": lat[c], "p99": lat[c] * 2.0}} + for c in COMPONENTS}, + "byte_provenance": {c: {"total_logical_bytes": nbytes} for c in COMPONENTS}, + "correctness": {"passed": passed}, + "routing": {"locality": {"cross_node_fraction": 0.5}}, + } + + +def _doc(rows, ep=2, mode="normal"): + case = {"ep": ep, "backend": "deepep-v2", "precision": "bf16", "phase": "decode", + "mode": mode, "suite": "s", "routing": "uniform"} + return { + "generated_at": "2026-07-25T22:12:55.760511+00:00", + "identity": {"attempt_ordinal": 1, "allocation_factors": {"run_id": "30177021271"}, + "case_factors": {"sku": "h100", "case": case}}, + "measurement": {"rows": rows}, + "outcome": {"status": "success"}, + } + + +def _linear(pairs, passed=True): + """Rows exactly on latency = 10us + bytes * 2e-6, i.e. alpha=10, beta_agg=500 GB/s.""" + return [_row(t, b, 10.0 + b * 2e-6, passed) for t, b in pairs] + + +LADDER = ((8, 1e6), (16, 2e6), (32, 3e6)) + + +class BandwidthMath(unittest.TestCase): + def test_fit_is_none_when_undefensible(self): + flat = [_row(t, b, 12.0) for t, b in LADDER] # slope <= 0 + self.assertIsNone(bandwidth.fit_alpha_beta(_doc(_linear(LADDER[:2])), "dispatch")) + self.assertIsNone(bandwidth.fit_alpha_beta(_doc(flat), "dispatch")) + + def test_noisy_ladder_withholds_beta(self): + # A positive slope through noise still fits; printing its beta once produced a + # physically impossible 1018 GB/s per GPU on a B200 at R2 = 0.29. + rows = [_row(t, b, lat) for t, b, lat in ( + (8, 1e6, 300.0), (16, 2e6, 40.0), (32, 3e6, 260.0), + (64, 4e6, 60.0), (128, 5e6, 320.0))] + fit = bandwidth.fit_alpha_beta(_doc(rows), "dispatch") + self.assertLess(fit.r2, bandwidth.FIT_MIN_R2) + self.assertFalse(fit.beta_is_reliable) + out = bandwidth.render([_doc(rows)]) + self.assertIn("beta=unreliable", out) + self.assertNotIn("GB/s alpha", out) # no number presented as measured + + def test_gate_failed_rung_excluded_from_fit_and_marked(self): + rows = _linear(LADDER) + [_row(64, 4e6, 999.0, passed=False)] + fit = bandwidth.fit_alpha_beta(_doc(rows), "dispatch") + self.assertEqual((fit.points, fit.excluded_rows), (3, 1)) + self.assertAlmostEqual(fit.beta_gbps, 250.0, places=4) # the corrupt rung didn't steer it + out = bandwidth.render([_doc(rows)]) + self.assertIn("[correctness FAILED]", out) + self.assertIn("excluded 1 gate-failed rung", out) + if __name__ == "__main__": unittest.main() diff --git a/experimental/CollectiveX/tests/test_roundtrip_staging.py b/experimental/CollectiveX/tests/test_roundtrip_staging.py deleted file mode 100644 index b6338fc50..000000000 --- a/experimental/CollectiveX/tests/test_roundtrip_staging.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -"""Contract for what the chained roundtrip measures. - -`stage` exists only for FP8 (`stage_device_work = self._fp8`), so charging it to the chained -roundtrip compares FP8 and BF16 through structurally different pipelines. Real stacks decide -this on quant-format match: SGLang's DeepEP dispatcher contains no dequant at all, and vLLM -returns the dispatched fp8 + scales untouched when `block_k == DEEPEP_QUANT_BLOCK_SIZE`, -dequantising only as a mismatch fallback. These tests pin both models. -""" -from __future__ import annotations - -import sys -import types -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path[:0] = [str(ROOT), str(ROOT / "bench")] - -import ep_backend # noqa: E402 - - -class _StubBackend(ep_backend.EPBackend): - """Records the call order; no device work.""" - - name = "stub" - - def __init__(self, stage_device_work: bool, fp8_consume: str, precision: str = "fp8"): - self.calls: list[str] = [] - self.stage_device_work = stage_device_work - self.fp8_consume = fp8_consume - self.precision = precision - - def create_buffer(self, spec): # pragma: no cover - unused - raise NotImplementedError - - def dispatch(self, problem): - self.calls.append("dispatch") - return types.SimpleNamespace(combine_input=None) - - def stage(self, problem, handle): - self.calls.append("stage") - handle.combine_input = "staged-by-stage" - - def combine(self, problem, handle): - self.calls.append(f"combine({handle.combine_input})") - return handle.combine_input - - def recv_tokens(self, handle): # pragma: no cover - unused - return 0 - - def inspect_dispatch(self, problem, handle): # pragma: no cover - unused - return {} - - def combine_transformed(self, problem, handle, transformed): # pragma: no cover - return transformed - - -class RoundtripStaging(unittest.TestCase): - def test_staged_input_keeps_the_conversion_out_of_the_chain(self): - b = _StubBackend(stage_device_work=True, fp8_consume="native") - b.run_roundtrip(object(), staged="pre-materialised") - self.assertEqual(b.calls, ["dispatch", "combine(pre-materialised)"]) - self.assertNotIn("stage", b.calls) - - def test_without_staged_input_the_stage_runs_inline(self): - # The fp8 `dequant` model takes this path, and so does BF16 — free for the adapters - # whose receive buffer is already the combine input, real work for mori/flashinfer-ep. - b = _StubBackend(stage_device_work=True, fp8_consume="dequant") - b.run_roundtrip(object()) - self.assertEqual(b.calls, ["dispatch", "stage", "combine(staged-by-stage)"]) - - def test_adapters_declare_where_their_combine_input_lives(self): - # A wrong attribute would silently leave the staged tensor unused, so the roundtrip - # would measure a combine over stale data. NCCL EP is the one that differs. - self.assertEqual(ep_backend.EPBackend.combine_input_attr, "combine_input") - b = _StubBackend(stage_device_work=True, fp8_consume="native") - b.combine_input_attr = "combine_input" - b.run_roundtrip(object(), staged="X") - self.assertEqual(b.calls[-1], "combine(X)") - - def test_default_models_the_native_path(self): - # deepseek-v3 block-fp8 hits vLLM's matched branch and SGLang's no-dequant path. - self.assertEqual(ep_backend.EPBackend.fp8_consume, "native") - - -class NativeStagingGate(unittest.TestCase): - """`stage_device_work` does NOT imply fp8, so the gate must check precision. - - MoRI sets `stage_device_work = self._fp8 or not self._external_input`, so its scale-up - kernels report True for BF16 too, and their stage() does a real copy into the registered - combine-input buffer. Gating on stage_device_work alone silently lifted that copy out of - the BF16 timed region -- a precision-asymmetric change of exactly the kind this file - exists to prevent. - """ - - def test_bf16_with_device_staging_keeps_the_stage_inline(self): - mori_intranode_bf16 = _StubBackend( - stage_device_work=True, fp8_consume="native", precision="bf16" - ) - self.assertFalse(mori_intranode_bf16.stages_fp8_natively) - mori_intranode_bf16.run_roundtrip(object()) - self.assertEqual( - mori_intranode_bf16.calls, ["dispatch", "stage", "combine(staged-by-stage)"] - ) - - def test_fp8_with_device_staging_lifts_the_conversion_out(self): - self.assertTrue( - _StubBackend( - stage_device_work=True, fp8_consume="native", precision="fp8" - ).stages_fp8_natively - ) - - def test_the_hatch_and_no_op_stages_never_take_the_fast_path(self): - self.assertFalse( # CX_FP8_CONSUME=dequant restores the inline stage - _StubBackend( - stage_device_work=True, fp8_consume="dequant", precision="fp8" - ).stages_fp8_natively - ) - self.assertFalse( # nccl-ep / bf16 deepep-v2: nothing to lift - _StubBackend( - stage_device_work=False, fp8_consume="native", precision="fp8" - ).stages_fp8_natively - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/experimental/CollectiveX/tests/test_runtime.py b/experimental/CollectiveX/tests/test_runtime.py index 01f377b7a..3120879e0 100644 --- a/experimental/CollectiveX/tests/test_runtime.py +++ b/experimental/CollectiveX/tests/test_runtime.py @@ -16,6 +16,7 @@ import tempfile import types import unittest +from unittest import mock RUNTIME = Path(__file__).resolve().parents[1] / "runtime" @@ -27,6 +28,7 @@ import config # noqa: E402 import stage # noqa: E402 import ep_harness # noqa: E402 (stdlib-only at module top) +import ep_backend # noqa: E402 (torch is imported lazily inside its methods) # configs/platform_config.json is shared by matrix scheduling, operator/network @@ -72,15 +74,6 @@ def test_every_platform_entry_is_complete_and_typed(self) -> None: class ProbeTests(unittest.TestCase): - def test_default_route_interface(self) -> None: - with tempfile.TemporaryDirectory() as directory: - route = Path(directory) / "route" - route.write_text( - "Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT\n" - "eth9 00000000 00000000 0003 0 0 0 00000000 0 0 0\n" - ) - self.assertEqual(probe.default_route_interface(route), "eth9") - def test_prepare_cache_is_private_and_reusable(self) -> None: with tempfile.TemporaryDirectory() as directory: first = Path(probe.prepare_cache(directory)) @@ -88,6 +81,17 @@ def test_prepare_cache_is_private_and_reusable(self) -> None: self.assertEqual(first, second) self.assertEqual(first.stat().st_mode & 0o777, 0o700) + def test_prepare_cache_bootstraps_a_missing_squash_dir(self) -> None: + # The probe runs before the first container import, so on a fresh pool squash_dir does + # not exist yet; a bare mkdir killed every b200-nscale leg of that pool's first sweep. + with tempfile.TemporaryDirectory() as directory: + parent = Path(directory) / "sqsh" + self.assertFalse(parent.exists()) + cache = Path(probe.prepare_cache(str(parent))) + self.assertTrue(cache.is_dir()) + self.assertEqual(cache.parent, parent.resolve()) + self.assertEqual(cache.stat().st_mode & 0o777, 0o700) + class ConfigTests(unittest.TestCase): def test_operator_config_emits_allowlisted_values(self) -> None: @@ -141,14 +145,6 @@ def test_operator_config_registry_only_emits_tracked_baseline(self) -> None: self.assertIn(b"COLLX_SQUASH_DIR\0/home/sa-shared/containers\0", payload) self.assertIn(b"COLLX_RDMA_DEVICES\0", payload) - def test_operator_config_registry_only_emits_image_for_secret_fed_sku(self) -> None: - # A SKU without tracked operator settings still gets its public image - # configuration; private scheduler values can arrive through the overlay. - payload = self._emit_registry_only("mi325x-tw") - self.assertIn(b"COLLX_IMAGE\0rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0701\0", payload) - self.assertIn(b"COLLX_IMAGE_PLATFORM\0linux/amd64\0", payload) - - class StageTests(unittest.TestCase): def test_create_copy_and_validate_cleanup(self) -> None: with tempfile.TemporaryDirectory() as directory: @@ -232,27 +228,6 @@ def test_healthy_fabric_emits_the_success_markers_the_launcher_extracts(self) -> self.assertEqual(self._captures(SOCKET_MARKER, lines), ["eth0"]) self.assertEqual(self._captures(LINK_MARKER, lines), ["roce"]) - def test_infiniband_link_layer_maps_to_the_launcher_token(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self._fabric(root, link_layer="InfiniBand") - rc, lines = self._run(root, root / "route") - self.assertEqual(rc, 0) - self.assertEqual(self._captures(LINK_MARKER, lines), ["infiniband"]) - - def test_socket_interface_resolves_from_default_route(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self._fabric(root) - route = root / "route" - route.write_text( - "Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT\n" - "eth0 00000000 00000000 0003 0 0 0 00000000 0 0 0\n" - ) - rc, lines = self._run(root, route, socket_names="") - self.assertEqual(rc, 0) - self.assertEqual(self._captures(SOCKET_MARKER, lines), ["eth0"]) - def test_inactive_port_emits_a_launcher_recognized_failure_marker(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -262,13 +237,88 @@ def test_inactive_port_emits_a_launcher_recognized_failure_marker(self) -> None: failures = [line for line in lines if re.search(FAILURE_MARKER, line)] self.assertTrue(any("rdma-port-1=inactive" in line for line in failures), failures) - def test_all_zero_gid_emits_gid_empty(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - self._fabric(root, gid="0000:0000:0000:0000:0000:0000:0000:0000") - rc, lines = self._run(root, root / "route") - self.assertEqual(rc, 1) - self.assertTrue(any("rdma-port-1=gid-empty" in line for line in lines), lines) +# config.py case-args is the single case→invocation codec: collx_run_shard decodes one +# null-delimited argv per case and hands it verbatim to bench/run_ep.py. Parse the +# emitted argv with the same parser shape run_ep builds so the two sides cannot +# drift — a flag the codec emits but run_ep does not declare (or vice versa) fails +# here instead of on a GPU allocation. +# logical_byte_provenance is where FP8 changes MEASUREMENT semantics (asymmetric +# per-direction byte counts), so its arithmetic and guards are pinned here on CPU. +try: + import torch as _torch +except Exception: # torch is absent in the CPU test image; these checks run on GPU CI + _torch = None + + +class ContainerImportRetry(unittest.TestCase): + """A failed container import is retried, because the failure is usually the storage blinking. + + The import writes tens of GB to operator-supplied squash storage, which on some clusters is a + SOFT-mounted network filesystem -- one that returns an error instead of blocking when its + transport drops. gb300's /data is NFSv3 over RDMA, and a transport gap there surfaces from + `mkdir` as "Protocol family not supported", which reads like a missing mount but is not: the + same node writes it fine minutes later. Run 31089556516 lost its gb300 shards that way, ~25 + minutes into each leg, so the import must not treat one such failure as terminal. + """ + + HARNESS = """ +set -u +export COLLX_IMAGE_PLATFORM=linux/amd64 +export COLLX_JOB_ROOT="$ROOT/job" +mkdir -p "$COLLX_JOB_ROOT" +mkdir -p "$ROOT/bin" "$ROOT/sqsh" +# Fake srun: appends one line per invocation and replays a scripted exit-code sequence. +cat > "$ROOT/bin/srun" <<'FAKE' +#!/bin/bash +echo call >> "$ROOT/calls" +n=$(wc -l < "$ROOT/calls" | tr -d ' ') +codes=($RC_SEQUENCE) +idx=$(( n - 1 )); [ $idx -ge ${#codes[@]} ] && idx=$(( ${#codes[@]} - 1 )) +exit ${codes[$idx]} +FAKE +chmod +x "$ROOT/bin/srun" +export PATH="$ROOT/bin:$PATH" +source "$COMMON" +sleep() { :; } # collapse the backoff +unsquashfs() { return 0; } # a present squash short-circuits the import +out="$(collx_ensure_squash_on_job 12345 "$ROOT/sqsh" some/image:tag)"; rc=$? +echo "RC=$rc" +echo "OUT=$out" +echo "CALLS=$(wc -l < "$ROOT/calls" 2>/dev/null | tr -d ' ' || echo 0)" +""" + + def _run(self, rc_sequence: str): + with tempfile.TemporaryDirectory() as root: + proc = subprocess.run( + ["bash", "-c", self.HARNESS], + env={ + **os.environ, "ROOT": root, "COMMON": str(RUNTIME / "common.sh"), + "RC_SEQUENCE": rc_sequence, "COLLX_IMPORT_ATTEMPTS": "3", + }, + capture_output=True, text=True, + ) + fields = dict( + line.split("=", 1) for line in proc.stdout.splitlines() if "=" in line + and line.split("=", 1)[0] in ("RC", "OUT", "CALLS") + ) + return fields, proc + + def test_a_transient_failure_is_retried_and_then_succeeds(self): + # Also the fixture's own control: if the job-root shape were wrong the function would + # fail before ever reaching srun, CALLS would be 0, and every assertion here would pass + # vacuously. Asserting the invocation count is what makes that impossible. + fields, proc = self._run("1 0") + self.assertEqual(fields.get("CALLS"), "2", proc.stdout + proc.stderr) + self.assertEqual(fields.get("RC"), "0", proc.stdout + proc.stderr) + # Callers capture stdout as the squash path, so nothing else may reach it. + self.assertTrue(fields.get("OUT", "").endswith(".sqsh"), fields) + + def test_an_architecture_mismatch_is_not_retried(self): + # rc 13 is the remote platform mismatch: a property of the case, not the moment, so + # retrying only delays the real message by two backoffs. + fields, proc = self._run("13 13 13") + self.assertEqual(fields.get("CALLS"), "1", proc.stdout + proc.stderr) + self.assertEqual(fields.get("RC"), "1", proc.stdout + proc.stderr) class StageContract(unittest.TestCase): @@ -308,6 +358,52 @@ def test_launcher_only_invokes_declared_subcommands_and_flags(self) -> None: except SystemExit: self.fail(f"common.sh invokes stage.py with an argv shape it rejects: {argv}") + # config.py accepts `exclude_nodes` for every SKU, but only the launcher named by that SKU's + # `launcher` field can turn it into salloc --exclude. When a launcher ignores it the key is + # accepted, exported, and silently dropped -- so a tray quarantined in the registry keeps + # getting scheduled and the config reads like a fix that never fired. That is exactly what + # launch_gb-nv.sh did: it built no allocation array at all. + def test_every_skus_denylist_reaches_its_launcher(self) -> None: + registry = json.loads( + (RUNTIME.parent / "configs" / "platform_config.json").read_text())["platforms"] + launchers = RUNTIME.parent / "launchers" + checked = 0 + for sku, platform in registry.items(): + if not platform.get("operator", {}).get("exclude_nodes"): + continue + script = launchers / f"launch_{platform['launcher']}.sh" + self.assertTrue(script.exists(), f"{sku} names a launcher that does not exist") + self.assertIn( + "COLLX_EXCLUDE_NODES", script.read_text(), + f"{sku} declares exclude_nodes but {script.name} never reads it, " + "so the denylist is silently discarded", + ) + checked += 1 + self.assertGreater(checked, 0, "no SKU declares exclude_nodes -- test proves nothing") + + # CX_FP8_CONSUME is read at class-body evaluation and fails closed on an unrecognised + # value, so the workflow must never hand it one. A blank dispatch input is the trap: it + # sets the variable to "" rather than leaving it unset, os.environ.get's default never + # applies, and every leg dies at import before any measurement. + def test_the_workflow_never_passes_an_invalid_fp8_consume(self) -> None: + workflow = ( + RUNTIME.parents[2] / ".github" / "workflows" / "collectivex-sweep.yml" + ) + if not workflow.exists(): # pragma: no cover - repo layout guard + self.skipTest("workflow not present in this checkout") + text = workflow.read_text() + if "CX_FP8_CONSUME" not in text: + self.skipTest("workflow does not set CX_FP8_CONSUME") + line = next(l for l in text.splitlines() if l.strip().startswith("CX_FP8_CONSUME:")) + self.assertIn( + "|| 'native'", line, + "a blank fp8_consume input must fall back to 'native'; passing it through empty " + "sets the variable to \"\" and the harness fails closed at import", + ) + for value in re.findall(r"'([a-z]*)'", line): + if value: + self.assertIn(value, ("native", "dequant"), value) + def test_contract_test_has_teeth(self) -> None: # A flag common.sh must never pass has to be rejected by the parser — this is the exact # failure (unrecognized arguments: --allow-parent-owner) the reconcile removed. @@ -331,7 +427,13 @@ class CaseArgvContract(unittest.TestCase): "scale_up_transport": "nvlink", "scale_out_transport": "rdma", "transport": "nvlink-rdma", "topology_class": "h200-nvlink-rdma", "hidden": 7168, "topk": 8, "experts": 256, "seed": 67, - "ladder": "1 2 4", "timing": "8:256:32", + "ladder": "1 2 4", + # The current producer shape: an object naming every knob (sweep_matrix emits this). + # The colon-string fixtures below are legacy shards, exercising _migrate_timing. + "timing": { + "iters_per_trial": 8, "trials_per_point": 256, "warmup_iters_per_trial": 32, + "chain_iters_per_trial": 128, "chain_trials_per_point": 4, "chain_drop": 16, + }, "case_id": "h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep16-uniform-bf16", "suite": "ep-core", "workload": "deepseek-v3", } @@ -379,134 +481,75 @@ def test_case_args_round_trips_through_the_run_ep_parser(self) -> None: self.assertEqual(args.version, 1) self.assertEqual(args.seed, self.CASE["seed"]) self.assertEqual((args.iters, args.trials, args.warmup), (8, 256, 32)) - self.assertEqual(args.out, "results/h200-dgxc_deepep-v2_bf16_decode_TS-c000.json") + self.assertEqual(args.out, f"results/{self.CASE['case_id']}_TS-c000.json") + + def test_a_legacy_colon_string_profile_still_decodes(self) -> None: + # Sweep `version` does not bump for the codec change, so a shard staged before it -- + # or one built by hand -- must still produce a runnable argv. Both legacy arities: + # six positional fields, and the pre-chain three whose chain knobs then come from + # run_ep's own defaults rather than being duplicated in the codec. + for profile, chain in (("8:256:32:128:4:16", (128, 4, 16)), ("8:256:32", None)): + with self.subTest(timing=profile): + args = self._run_ep_parser().parse_args(self._case_argv( + ["16", "2", "8", "8"], case={**self.CASE, "timing": profile}, + )) + self.assertEqual((args.iters, args.trials, args.warmup), (8, 256, 32)) + if chain: + self.assertEqual( + (args.chain_iters, args.chain_trials, args.chain_drop), chain + ) + continue + for flag in ("chain_iters", "chain_trials", "chain_drop"): + self.assertIsInstance(getattr(args, flag), int) + # A chain that drops everything it measured has no samples left to reduce. + self.assertGreater(args.chain_iters, args.chain_drop) + self.assertGreater(args.chain_trials, 0) + + def test_a_malformed_timing_profile_cannot_reach_a_run(self) -> None: + # Every rejection path in one place. Objects: an unknown, renamed or missing key must be + # as fatal as a bad string arity, or a renamed knob silently falls back to run_ep's + # default. Strings: three fields is the pre-chain profile and six the chain profile; any + # other length is a shard built against a codec that no longer exists. + for timing in ( + {"iters_per_trial": 8}, {**self.CASE["timing"], "extra": 1}, {}, + "8:256", "8:256:32:128", "8:256:32:128:4:16:2", "", + ): + with self.subTest(timing=timing): + with self.assertRaises(subprocess.CalledProcessError): + self._case_argv(["16", "2", "8", "8"], case={**self.CASE, "timing": timing}) + # Types are NOT checked by the codec -- as has always been true for iters/trials/warmup + # -- so the property is that the argv it emits still cannot parse into a run. + argv = self._case_argv( + ["16", "2", "8", "8"], case={**self.CASE, "timing": "8:256:32:128:4:x"}, + ) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + self._run_ep_parser().parse_args(argv) def test_case_args_fails_closed_on_placement_mismatch(self) -> None: with self.assertRaises(subprocess.CalledProcessError): self._case_argv(["8", "1", "8", "8"]) - def test_low_latency_case_round_trips_through_the_run_ep_parser(self) -> None: - # A low-latency decode EP8 case flows through the same codec; run_ep's --mode - # choices must accept "low-latency" or the leg dies before allocation. - ll_case = { - **self.CASE, - "mode": "low-latency", "phase": "decode", - "ep": 8, "nodes": 1, "gpus_per_node": 8, "scale_up_domain": 8, - "scope": "scale-up", "scale_up_transport": "nvlink", - "scale_out_transport": "", "transport": "nvlink", - "topology_class": "h200-nvlink-island", "ladder": "1 2 4 8", - "case_id": "h200-dgxc-deepep-v2-deepseek-v3-low-latency-decode-ep8-uniform-bf16", - } - argv = self._case_argv(["8", "1", "8", "8"], case=ll_case) - args = self._run_ep_parser().parse_args(argv) - self.assertEqual((args.mode, args.phase, args.scope), ("low-latency", "decode", "scale-up")) - self.assertEqual(args.case_id, ll_case["case_id"]) - - def test_uccl_ep_case_round_trips_through_the_run_ep_parser(self) -> None: - # A uccl-ep case flows through the same generic codec; run_ep's --backend choices - # must accept "uccl-ep" and the result filename must carry the backend token so a - # uccl-ep leg never collides with the deepep-v2/mori legs of the same cell. - uccl_case = { - **self.CASE, - "backend": "uccl-ep", - "case_id": "h200-dgxc-uccl-ep-deepseek-v3-normal-decode-ep16-uniform-bf16", - } - argv = self._case_argv(["16", "2", "8", "8"], case=uccl_case) - args = self._run_ep_parser().parse_args(argv) - self.assertEqual(args.backend, "uccl-ep") - self.assertEqual(args.case_id, uccl_case["case_id"]) - self.assertEqual(args.out, "results/h200-dgxc_uccl-ep_bf16_decode_TS-c000.json") - - def test_nccl_ep_case_round_trips_through_the_run_ep_parser(self) -> None: - # A nccl-ep case flows through the same generic codec; run_ep's --backend choices must - # accept "nccl-ep" and the result filename must carry the backend token so a nccl-ep leg - # never collides with the deepep-v2/uccl-ep legs of the same cell. BF16 only. - nccl_case = { - **self.CASE, - "backend": "nccl-ep", - "case_id": "h200-dgxc-nccl-ep-deepseek-v3-normal-decode-ep16-uniform-bf16", - } - argv = self._case_argv(["16", "2", "8", "8"], case=nccl_case) - args = self._run_ep_parser().parse_args(argv) - self.assertEqual(args.backend, "nccl-ep") - self.assertEqual(args.case_id, nccl_case["case_id"]) - self.assertEqual(args.out, "results/h200-dgxc_nccl-ep_bf16_decode_TS-c000.json") - - def test_flashinfer_ep_case_round_trips_through_the_run_ep_parser(self) -> None: - # A flashinfer-ep case flows through the same generic codec; run_ep's --backend - # choices must accept "flashinfer-ep" and the result filename must carry the backend - # token so it never collides with the deepep-v2/nccl-ep legs of the same cell. - # The codec is SKU-agnostic, so this reuses the shared h200 fixture like its - # siblings; that flashinfer-ep is GB-only is a registry fact, pinned separately by - # test_matrix.test_flashinfer_ep_rollout_shape. - flashinfer_case = { - **self.CASE, - "backend": "flashinfer-ep", - "case_id": "h200-dgxc-flashinfer-ep-deepseek-v3-normal-decode-ep16-uniform-bf16", - } - argv = self._case_argv(["16", "2", "8", "8"], case=flashinfer_case) - args = self._run_ep_parser().parse_args(argv) - self.assertEqual(args.backend, "flashinfer-ep") - self.assertEqual(args.case_id, flashinfer_case["case_id"]) - self.assertEqual( - args.out, "results/h200-dgxc_flashinfer-ep_bf16_decode_TS-c000.json" - ) - - def test_mirrored_backend_choices_match_run_ep(self) -> None: - """The mirror is only worth having if it cannot drift from the real parser. - - Kept in sync by convention it has now drifted twice — a backend was added to - run_ep.py's choices while this fixture kept the old list, so a case_id that the real - CLI accepts raised SystemExit here. Read the real list out of the source (AST, not - import: importing run_ep pulls in torch and the vendor EP libraries) and compare. - """ - tree = ast.parse((BENCH / "run_ep.py").read_text()) - real = [ - [element.value for element in keyword.value.elts] - for node in ast.walk(tree) - if isinstance(node, ast.Call) - for argument in node.args - if isinstance(argument, ast.Constant) and argument.value == "--backend" - for keyword in node.keywords - if keyword.arg == "choices" - ] - self.assertEqual(len(real), 1, "expected exactly one --backend choices list") - mirrored = next( - action.choices - for action in self._run_ep_parser()._actions - if action.dest == "backend" - ) - self.assertEqual(sorted(real[0]), sorted(mirrored)) - + def test_each_backend_round_trips_through_the_run_ep_parser(self) -> None: + # The codec is backend-agnostic, so one loop replaces three near-identical tests: + # run_ep's --backend choices must accept each name, and the filename must carry the + # backend token or two legs of one cell collide in results/. That flashinfer-ep is + # GB-only is a registry fact, pinned in test_matrix. + for backend in ("uccl-ep", "nccl-ep", "flashinfer-ep"): + with self.subTest(backend=backend): + case = { + **self.CASE, "backend": backend, + "case_id": f"h200-dgxc-{backend}-deepseek-v3-normal-decode-ep16-uniform-bf16", + } + args = self._run_ep_parser().parse_args( + self._case_argv(["16", "2", "8", "8"], case=case) + ) + self.assertEqual(args.backend, backend) + self.assertEqual(args.case_id, case["case_id"]) + self.assertEqual(args.out, f"results/{case['case_id']}_TS-c000.json") # logical_byte_provenance is where FP8 changes MEASUREMENT semantics (asymmetric # per-direction byte counts), so its arithmetic and guards are pinned here on CPU. class LogicalByteProvenanceTests(unittest.TestCase): - def test_bf16_default_is_two_bytes_per_value_no_scales(self) -> None: - got = ep_harness.logical_byte_provenance(logical_copies=10, hidden=7168) - self.assertEqual(got["activation_data_bytes"], 10 * 7168 * 2) - self.assertEqual(got["scale_bytes"], 0) - self.assertEqual(got["total_logical_bytes"], 10 * 7168 * 2) - - def test_fp8_blockwise_dispatch_is_one_byte_plus_per_copy_scales(self) -> None: - # DeepEP FP8 dispatch: 1 byte/value + ceil(hidden/128)*4 FP32 scale bytes/copy. - scale_per_copy = ((7168 + 127) // 128) * 4 # 224 - got = ep_harness.logical_byte_provenance( - logical_copies=10, hidden=7168, value_bytes=1, - scale_bytes_per_copy=scale_per_copy, - ) - self.assertEqual(got["activation_data_bytes"], 10 * 7168) - self.assertEqual(got["scale_bytes"], 10 * scale_per_copy) - self.assertEqual(got["total_logical_bytes"], 10 * 7168 + 10 * scale_per_copy) - - def test_fp8_direct_cast_dispatch_is_one_byte_no_scales(self) -> None: - # MoRI's scale-free e4m3 cast: 1 byte/value, no scale payload. - got = ep_harness.logical_byte_provenance( - logical_copies=10, hidden=7168, value_bytes=1, scale_bytes_per_copy=0, - ) - self.assertEqual(got["activation_data_bytes"], 10 * 7168) - self.assertEqual(got["scale_bytes"], 0) - def test_roundtrip_is_the_per_field_sum_of_dispatch_and_combine(self) -> None: # run_sweep assembles the roundtrip as the per-field sum of an FP8 dispatch and a # BF16 combine; the direction bytes differ, so it is not 2x a single direction. @@ -544,6 +587,18 @@ def test_mode_allowed_semantics(self) -> None: {"weighted-kernel-sum", "unweighted-rank-sum"}, ) + def test_oracle_modeled_contract_pairs(self) -> None: + # receive_layout and combine_weight_semantics are independent declarations; + # only these two pairings have an expected-combine model, and run_sweep fails + # closed on the other two rather than verify against the wrong oracle. + self.assertEqual( + ep_harness.ORACLE_MODELED_CONTRACTS, + { + ("token-rank", "unweighted-rank-sum"), + ("token-expert", "weighted-kernel-sum"), + }, + ) + try: import torch as _torch @@ -564,19 +619,6 @@ def _problem(self, weight_scale: float = 1.0): weights = (torch.rand(4, 2, dtype=torch.float32) + 0.1) * weight_scale return types.SimpleNamespace(x=x, topk_idx=idx, topk_weights=weights) - def test_transform_drops_the_gate_under_weighted_kernel_sum(self): - torch = _torch - payload = torch.randn(3, 64, dtype=torch.bfloat16) - ids = torch.tensor([[2, -1], [5, -1], [7, -1]], dtype=torch.int64) - low = ep_harness._expert_transform( - torch, payload, ids, torch.full((3, 2), 0.2), "weighted-kernel-sum" - ) - high = ep_harness._expert_transform( - torch, payload, ids, torch.full((3, 2), 0.9), "weighted-kernel-sum" - ) - # Unit coefficient: the staged value cannot depend on the gate magnitude. - self.assertTrue(torch.equal(low, high)) - def test_transform_folds_the_gate_under_unweighted_rank_sum(self): torch = _torch payload = torch.randn(3, 64, dtype=torch.bfloat16) @@ -590,21 +632,6 @@ def test_transform_folds_the_gate_under_unweighted_rank_sum(self): # The gate IS in the transform here, so a larger weight changes the staged value. self.assertFalse(torch.equal(low, high)) - def test_expected_combine_is_linear_in_the_gate_under_weighted_kernel_sum(self): - torch = _torch - p = self._problem(1.0) - p2 = types.SimpleNamespace( - x=p.x, topk_idx=p.topk_idx, topk_weights=p.topk_weights * 2 - ) - base = ep_harness._expected_transformed_combine( - torch, p, 4, 8, "weighted-kernel-sum" - ) - doubled = ep_harness._expected_transformed_combine( - torch, p2, 4, 8, "weighted-kernel-sum" - ) - # Same routing/activations, gate x2 -> expected x2 (the kernel applies the gate). - self.assertTrue(torch.allclose(doubled, base * 2, atol=1e-3, rtol=1e-3)) - def test_unknown_semantics_fail_closed(self): torch = _torch with self.assertRaises(ValueError): @@ -643,16 +670,187 @@ def test_differs_from_both_rejected_models(self): self.assertNotEqual(self._tree(values), 1.015625) # FP32 accumulate, narrow once self.assertNotEqual(self._tree(values), 1.0) # sequential BF16 accumulate - def test_a_rank_claimed_by_an_earlier_slot_contributes_once(self): +@unittest.skipUnless(_torch is not None, "quantize-identity checks require torch") +class FusedQuantizeGate(unittest.TestCase): + """The oracle's payload gate compares the sender's [T, hidden] quantize against the oracle's + [receive_count, hidden] one, so a fused callable must be bit-identical and per-row invariant.""" + + @staticmethod + def _fuse(mode, eager): + # Both methods read only `self.mode`, so call them unbound rather than build a backend. + return ep_backend.EPBackend.fused_quantize(types.SimpleNamespace(mode=mode), eager) + + @staticmethod + def _check(eager, fused, x): + return ep_backend.EPBackend.assert_quantize_identity( + types.SimpleNamespace(mode="normal"), eager, fused, x + ) + + def test_low_latency_keeps_the_eager_helper(self): + # Its dispatch kernel quantises internally and the oracle gate is pinned to those bits, + # so a compiled callable would red every LL fp8 cell without touching timing. + def eager(x): + return x, x + self.assertIs(self._fuse("low-latency", eager), eager) + + def test_identity_check_rejects_a_divergent_callable(self): torch = _torch - # Both top-k slots route to rank 0; the kernel blanks the later slot in place. - destination = torch.zeros((1, 2), dtype=torch.int64) - messages = torch.full((1, 1, 1), 0.5) - combined = ep_harness._topk_slot_tree_combine( - torch, destination, torch.ones_like(destination, dtype=torch.bool), - messages, torch.bfloat16, + x = torch.randn(8, 256, dtype=torch.bfloat16) + + def eager(t): + return t.to(torch.float8_e4m3fn), t.float().abs().amax(dim=1) + + def divergent(t): + values, scales = eager(t) + return values, scales + 1 # one differing scale is enough to red a cell + with self.assertRaises(RuntimeError): + self._check(eager, divergent, x) + +class GpuHealthProbe(unittest.TestCase): + """Reject an allocation holding a throttled GPU before it burns the wall-clock guard: one + clamped device paces every rank (a B200 at 120 MHz against 1965 MHz ran a case 17x slower).""" + + HEALTHY = "\n".join(f"{i}, Not Active, Not Active, 3{i} " for i in range(8)) + + def _swap(self, line_in: str, line_out: str) -> str: + self.assertIn(line_in, self.HEALTHY) # guard the fixture against silent drift + return self.HEALTHY.replace(line_in, line_out) + + def test_a_clamped_gpu_is_rejected_by_either_signal(self): + # Throttle flags and temperature are INDEPENDENT signals: the flag can clear between + # samples while the fault persists, so heat alone must reject, and either flag alone is + # enough. The healthy fixture is the negative control -- a substring search for "Active" + # also matches "Not Active", which is the bug this shape guards. + self.assertEqual(probe.gpu_health_faults(self.HEALTHY), []) + for gpu, cells in ( + (7, "7, Active, Active, 93 "), (7, "7, Active, Not Active, 88 "), + (7, "7, Not Active, Active, 88 "), (3, "3, Not Active, Not Active, 95 "), + ): + with self.subTest(cells=cells): + faults = probe.gpu_health_faults( + self._swap(f"{gpu}, Not Active, Not Active, 3{gpu} ", cells) + ) + self.assertEqual(len(faults), 1) + self.assertIn(f"gpu {gpu}", faults[0]) + + def test_unreadable_output_fails_open(self): + # Blocking legs when the hardware cannot be read is worse than the fault being sought. + for output in ("", "nonsense\n", "1, Not Active\n", self.HEALTHY.replace("32 ", "[N/A] ")): + with self.subTest(output=output[:20]): + self.assertEqual(probe.gpu_health_faults(output), []) + + def _run_validate(self, csv: str, has_smi: bool = True): + """Drive validate_gpu_health with a stubbed nvidia-smi; returns (exit_code, stdout).""" + import shutil + real_which = shutil.which + shutil.which = (lambda name: "/usr/bin/nvidia-smi") if has_smi else (lambda name: None) + + class FakeSubprocess: + SubprocessError = subprocess.SubprocessError + + @staticmethod + def run(*args, **kwargs): + return types.SimpleNamespace(stdout=csv) + + sys.modules["subprocess"] = FakeSubprocess + captured = io.StringIO() + try: + with contextlib.redirect_stdout(captured): + probe.validate_gpu_health() + code = 0 + except SystemExit as exit_: + code = exit_.code + finally: + sys.modules["subprocess"] = subprocess + shutil.which = real_which + return code, captured.getvalue() + + def test_a_fault_exits_nonzero_and_names_the_gpu(self): + code, out = self._run_validate( + self._swap("7, Not Active, Not Active, 37 ", "7, Active, Active, 93 ") ) - self.assertEqual(combined.item(), 0.5) + self.assertEqual(code, 1) + self.assertIn("gpu-health-fault gpu 7", out) + self.assertNotIn("gpu-health-checked", out) + + def test_the_temperature_spread_is_reported_but_never_gated(self): + # The signal no gate can see: an H100 engages software thermal slowdown at ~86-87 C, so + # a clamped one never crosses the 90 C limit and the measured fault showed only as an + # idle outlier (55 C against ~30 C). Reported so a human can act, deliberately not gated, + # and absent rather than wrong when the output cannot be read. + sick = self._swap("3, Not Active, Not Active, 33 ", "3, Not Active, Not Active, 55 ") + self.assertEqual(probe.gpu_temperature_spread(sick), (55, 35, 20)) + hottest, median, spread = probe.gpu_temperature_spread(self.HEALTHY) + self.assertEqual((hottest, median), (37, 34)) # 8 temps -> median is index 4 + self.assertLess(spread, 10) + code, out = self._run_validate(sick) + self.assertEqual(code, 0) + self.assertIn("spread=20C", out) + for output in ("", "nonsense\n", self.HEALTHY.replace("33 ", "[N/A] ")): + with self.subTest(output=output[:16]): + result = probe.gpu_temperature_spread(output) + self.assertTrue(result is None or result[2] < 10) + + +class LowLatencyCapDecoupling(unittest.TestCase): + """The LL receive size and the measured ladder must stay two numbers -- sizing the receive + from `max(ladder)` would shift every rung. Driven through the adapter with deep_ep stubbed, + so the constants are exercised rather than read out of the syntax tree.""" + + @staticmethod + def _adapter(): + """Import ep_deepep_v2 with its vendor dependency stubbed out.""" + torch_stub = types.ModuleType("torch") + torch_stub.bfloat16 = torch_stub.float32 = torch_stub.int64 = "dtype" + torch_stub.distributed = types.SimpleNamespace(group=types.SimpleNamespace(WORLD=None)) + # The module decorates helpers at import time; pass them through untouched. + torch_stub.compile = lambda *a, **k: (a[0] if a else (lambda fn: fn)) + torch_stub._dynamo = types.SimpleNamespace(config=types.SimpleNamespace()) + deep_ep = types.ModuleType("deep_ep") + deep_ep.Buffer = type("Buffer", (), {}) + # The adapter imports ElasticBuffer by name and fails closed without it. + deep_ep.ElasticBuffer = type("ElasticBuffer", (), {}) + stubs = { + "torch": torch_stub, "torch.distributed": torch_stub.distributed, + "deep_ep": deep_ep, + } + with mock.patch.dict(sys.modules, stubs): + import importlib + import ep_deepep_v2 + return importlib.reload(ep_deepep_v2) + + def test_buffer_cap_tracks_the_ladder_constant(self): + # Patching the constant and watching the return move proves it reads the constant, which + # a literal that merely happens to equal it today would not. + module = self._adapter() + backend = module.DeepEPV2Backend.__new__(module.DeepEPV2Backend) + backend.mode = "low-latency" + with mock.patch.object(module, "_LL_LADDER_CAP", 64): + self.assertEqual(backend.buffer_cap(None), 64) + backend.mode = "normal" + self.assertIsNone(backend.buffer_cap(None)) + + def test_the_receive_is_sized_from_the_buffer_cap_not_the_ladder(self): + # The regression this guards would silently re-baseline every LL row: clamping the + # measured ladder must not shrink the receive the remaining rungs are measured against. + module = self._adapter() + source = (BENCH / "ep_deepep_v2.py").read_text() + sized = source.split("def create_buffer", 1)[1].split("def ", 1)[0] + self.assertIn("_LL_BUFFER_CAP", sized) + self.assertNotIn("_LL_LADDER_CAP", sized) + self.assertNotEqual(module._LL_BUFFER_CAP, None) + + def test_a_clamped_ladder_is_recorded_in_the_artifact_not_only_on_stdout(self): + # The clamp must reach the artifact: a rank-0 stdout NOTE alone leaves a document that + # measured 8 rungs indistinguishable from one that measured 9. Asserted on a real + # emitted document rather than on the presence of key literals in the source. + sys.path.insert(0, str(RUNTIME.parent / "tests")) + import test_chain + workload = test_chain.drive().doc["workload"] + for required in ("ladder_measured", "ladder_dropped", "ladder_cap"): + self.assertIn(required, workload, f"the emitted record must include {required}") + self.assertEqual(workload["ladder_measured"], list(test_chain.LADDER)) + if __name__ == "__main__": unittest.main()