diff --git a/packages/app/cypress/e2e/csv-export-overlay.cy.ts b/packages/app/cypress/e2e/csv-export-overlay.cy.ts index b43295c3..0d0034c9 100644 --- a/packages/app/cypress/e2e/csv-export-overlay.cy.ts +++ b/packages/app/cypress/e2e/csv-export-overlay.cy.ts @@ -35,9 +35,10 @@ function readCapturedCsv(): Cypress.Chainable { describe('Inference CSV export with an unofficial-run overlay', () => { before(() => { interceptOverlayRun(); - // The agentic default mode is E2E Normalized Interactivity (which suppresses overlays and - // fetches derived metrics) — stub the fetch, then switch to the - // Interactivity mode this suite's overlay assertions rely on. + // Agentic charts default to Interactivity, where the overlay renders. The + // derived-metrics fetch now happens only under E2E Normalized Interactivity, + // so the stub below is a guard against a stray request rather than a + // dependency of this suite. interceptDerivedAgenticMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, { onBeforeLoad(win) { @@ -47,12 +48,11 @@ describe('Inference CSV export with an unofficial-run overlay', () => { }, }); cy.wait('@unofficialRun'); - // Interactivity is nested under the Advanced menu on agentic charts. - cy.get('[data-testid="x-axis-mode-advanced"]').click(); - cy.get('[data-testid="x-axis-mode-interactivity"]').click(); - cy.get('[data-testid="x-axis-mode-advanced"]') - .should('have.attr', 'data-state', 'active') - .and('contain.text', 'Interactivity'); + // Every x-axis metric is a top-level tab on agentic charts, and Interactivity + // is the default — clicked anyway so the suite does not depend on that. + cy.get('[data-testid="x-axis-mode-interactivity"]') + .click() + .should('have.attr', 'data-state', 'active'); cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should('exist'); }); diff --git a/packages/app/cypress/e2e/overlay-legend-remove.cy.ts b/packages/app/cypress/e2e/overlay-legend-remove.cy.ts index d52df396..3be788fb 100644 --- a/packages/app/cypress/e2e/overlay-legend-remove.cy.ts +++ b/packages/app/cypress/e2e/overlay-legend-remove.cy.ts @@ -22,9 +22,10 @@ describe('Official legend X works while an unofficial overlay is loaded', () => // Use distinct hardware so the engine-comparison exclusion policy does // not resolve the official and unofficial rows as one competing family. interceptOverlayRun({ overlayHardware: 'h100' }); - // The agentic default mode is E2E Normalized Interactivity (which suppresses overlays and - // fetches derived metrics) — stub the fetch, then switch to Interactivity - // where the overlay renders. + // Agentic charts default to Interactivity, where the overlay renders. The + // derived-metrics fetch now happens only under E2E Normalized Interactivity, + // so the stub below is a guard against a stray request rather than a + // dependency of this suite. interceptDerivedAgenticMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, { onBeforeLoad(win) { @@ -33,9 +34,11 @@ describe('Official legend X works while an unofficial overlay is loaded', () => }, }); cy.wait('@unofficialRun'); - // Interactivity is nested under the Advanced menu on agentic charts. - cy.get('[data-testid="x-axis-mode-advanced"]').click(); - cy.get('[data-testid="x-axis-mode-interactivity"]').click(); + // Every x-axis metric is a top-level tab on agentic charts, and Interactivity + // is the default — clicked anyway so the suite does not depend on that. + cy.get('[data-testid="x-axis-mode-interactivity"]') + .click() + .should('have.attr', 'data-state', 'active'); cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should( 'have.length', @@ -43,6 +46,16 @@ describe('Official legend X works while an unofficial overlay is loaded', () => ); }); + // Cypress clears intercepts between tests, so the derived-metrics stub is + // re-registered per test rather than once in `before`. Until #736 the + // agentic default was E2E Normalized Interactivity, so the fetch happened + // during `before` while the stub was still alive and React Query held the + // result for the rest of the spec. The default no longer fetches, so any + // later switch into that mode issues a fresh request. + beforeEach(() => { + interceptDerivedAgenticMetrics(); + }); + it('shows official points and an official legend entry initially', () => { cy.get('[data-testid="inference-chart-display"] svg .dot-group').should(($dots) => { expect(countVisible($dots), 'visible official points').to.be.greaterThan(0); @@ -69,10 +82,12 @@ describe('Official legend X works while an unofficial overlay is loaded', () => cy.get('[data-testid="inference-chart-display"] svg .dot-group').should(($dots) => { expect(countVisible($dots), 'visible official points after remove').to.eq(0); }); - // The overlay series is untouched. Optimal Only still hides its trace-less - // points because unofficial rows cannot join the canonical frontier. + // The overlay series is untouched — hiding an official SKU must not disturb + // it. All five overlay points stay visible under Optimal Only: they are + // non-dominated on the interactivity axes, and since #736 lacking persisted + // traces no longer excludes them from the frontier. cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(($pts) => { - expect(countVisible($pts), 'visible overlay X markers').to.eq(0); + expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length); }); // Inactive row: the hover affordance flips to the "+" restore indicator // (explicit "clicking the name brings it back"), and the Hide X is gone. diff --git a/packages/app/cypress/e2e/overlay-optimal-only.cy.ts b/packages/app/cypress/e2e/overlay-optimal-only.cy.ts index 52056780..f9f388c4 100644 --- a/packages/app/cypress/e2e/overlay-optimal-only.cy.ts +++ b/packages/app/cypress/e2e/overlay-optimal-only.cy.ts @@ -1,22 +1,31 @@ /** - * Unofficial runs do not have persisted request traces, so they cannot join - * the canonical E2E Normalized Interactivity frontier. Optimal Only must hide - * them on every AgentX axis; Show All remains the explicit way to inspect them. + * Optimal Only must filter overlay (unofficial-run) points by the same rule it + * applies to official ones: the Pareto frontier of the *selected* axes. + * + * Until #736 the rule was different — unofficial runs have no persisted request + * traces, so they could never join the canonical E2E Normalized Interactivity + * frontier and Optimal Only hid every one of them. That gating is gone, so an + * overlay point now survives exactly when it is non-dominated on the axes on + * screen, and Show All remains the way to inspect the rest. */ import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e'; import { countVisible, + DOMINATED_CONFIG, interceptOverlayRun, OVERLAY_RUN_ID, REAL_CONFIGS, } from '../support/overlay-fixtures'; -describe('Overlay points follow canonical Optimal Only policy (agentic interactivity)', () => { +// The five real configs are all non-dominated on the interactivity axes, so a +// sixth, deliberately dominated point is what gives Optimal Only something to +// remove. Without it this suite would assert the same count either way and pass +// no matter what the filter did. +const OVERLAY_CONFIGS = [...REAL_CONFIGS, DOMINATED_CONFIG]; + +describe('Overlay points follow Optimal Only on the selected axes', () => { before(() => { - interceptOverlayRun(); - // The agentic default mode is E2E Normalized Interactivity (which suppresses overlays and - // fetches derived metrics) — stub the fetch, then switch to the - // Interactivity mode this suite is about. + interceptOverlayRun({ overlayConfigs: OVERLAY_CONFIGS }); interceptDerivedAgenticMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, { onBeforeLoad(win) { @@ -25,30 +34,37 @@ describe('Overlay points follow canonical Optimal Only policy (agentic interacti }, }); cy.wait('@unofficialRun'); - // Interactivity is nested under the Advanced menu on agentic charts. - cy.get('[data-testid="x-axis-mode-advanced"]').click(); - cy.get('[data-testid="x-axis-mode-interactivity"]').click(); + // Every x-axis metric is a top-level tab on agentic charts, and Interactivity + // is the default — clicked anyway so the suite does not depend on that. + cy.get('[data-testid="x-axis-mode-interactivity"]') + .click() + .should('have.attr', 'data-state', 'active'); cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); - cy.get('[data-testid="x-axis-mode-advanced"]') - .should('have.attr', 'data-state', 'active') - .and('contain.text', 'Interactivity'); + // All six are rendered; visibility is what Optimal Only changes. cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should( 'have.length', - REAL_CONFIGS.length, + OVERLAY_CONFIGS.length, ); }); - it('hides trace-less overlay points in the default Optimal Only view', () => { + // Cypress clears intercepts between tests, so the derived-metrics stub is + // re-registered per test rather than once in `before`. Until #736 the agentic + // default was E2E Normalized Interactivity, so the fetch happened during + // `before` while the stub was still alive and React Query held the result for + // the rest of the spec. The default no longer fetches. + beforeEach(() => { + interceptDerivedAgenticMetrics(); + }); + + it('drops only the dominated overlay point in the default Optimal Only view', () => { cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'checked'); - // The deterministic derived-metric stub puts all five official rows on the - // canonical frontier. cy.get('[data-testid="inference-chart-display"] svg .dot-group').should(($dots) => { expect(countVisible($dots), 'visible official points').to.eq(REAL_CONFIGS.length); }); - // Overlay rows have no persisted trace ids and therefore no canonical - // frontier membership. + // Five of six survive: every real config is non-dominated on these axes, and + // the overlay's trace-less rows are no longer excluded for lacking traces. cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(($pts) => { - expect(countVisible($pts), 'visible overlay X markers').to.eq(0); + expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length); }); }); @@ -56,15 +72,15 @@ describe('Overlay points follow canonical Optimal Only policy (agentic interacti cy.get('#scatter-hide-non-optimal').click(); cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'unchecked'); cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(($pts) => { - expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length); + expect(countVisible($pts), 'visible overlay X markers').to.eq(OVERLAY_CONFIGS.length); }); }); - it('re-hides trace-less overlay points when Optimal Only is re-enabled', () => { + it('re-drops the dominated overlay point when Optimal Only is re-enabled', () => { cy.get('#scatter-hide-non-optimal').click(); cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'checked'); cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(($pts) => { - expect(countVisible($pts), 'visible overlay X markers').to.eq(0); + expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length); }); }); }); diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index f62aed38..da5c47bc 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -124,25 +124,23 @@ const interceptFixedSequenceData = () => { }; /** - * On agentic charts, Interactivity / E2E Latency / TTFT live inside the - * "Advanced" menu rather than as top-level tabs, so they must be revealed - * before they can be clicked. The menu closes on select, so the post-select - * assertion targets the trigger, which carries the active state and the - * selected metric's label. + * Every x-axis metric is a top-level tab, on agentic charts as well as fixed + * sequences — #736 removed the "Advanced" popover that used to hide + * Interactivity / E2E Latency / TTFT behind a trigger. The tab itself now + * carries both the selected state and the metric's label. */ -function selectAdvancedXAxisMode(mode: 'interactivity' | 'e2e' | 'ttft', label: string) { - cy.get('[data-testid="x-axis-mode-advanced"]').click(); +function selectXAxisMode(mode: 'interactivity' | 'e2e' | 'ttft', label: string) { cy.get(`[data-testid="x-axis-mode-${mode}"]`).click(); - cy.get('[data-testid="x-axis-mode-advanced"]') - .should('have.attr', 'data-state', 'active') + cy.get(`[data-testid="x-axis-mode-${mode}"]`) + .should('have.attr', 'aria-selected', 'true') .and('contain.text', label); } describe('X-Axis Mode Toggle (inference chart)', () => { before(() => { interceptAgenticData(); - // The agentic default mode is E2E Normalized Interactivity, which fetches derived metrics - // on first render — stub them before the visit. + // Agentic defaults to Interactivity, which needs no derived metrics. The stub + // covers the first switch into E2E Normalized Interactivity further down. interceptDerivedAgenticMetrics(); cy.visit('/inference?i_seq=agentic-traces', { onBeforeLoad(win) { @@ -154,34 +152,50 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); }); - it('shows E2E Normalized Interactivity by default for the agentic view, as the leftmost option', () => { + // Cypress clears intercepts between tests, so the derived-metrics stub is + // re-registered per test rather than once in `before`. Until #736 the + // agentic default was E2E Normalized Interactivity, so the fetch happened + // during `before` while the stub was still alive and React Query held the + // result for the rest of the spec. The default no longer fetches, so any + // later switch into that mode issues a fresh request. + beforeEach(() => { + interceptDerivedAgenticMetrics(); + }); + + it('defaults the agentic view to Interactivity, with all four modes as flat tabs', () => { cy.get('[data-testid="scenario-selector"]').should('contain.text', 'Agentic Traces'); - // The three per-request latency modes are nested under Advanced on agentic. - cy.get('[data-testid="x-axis-mode-ttft"]').should('not.exist'); - cy.get('[data-testid="x-axis-mode-e2e"]').should('not.exist'); - cy.get('[data-testid="x-axis-mode-interactivity"]').should('not.exist'); - cy.get('[data-testid="x-axis-mode-advanced"]') - .should('be.visible') - .and('have.attr', 'data-state', 'inactive'); - cy.get('[data-testid="x-axis-mode-e2e-normalized-interactivity"]') - .should('be.visible') - .and('have.attr', 'aria-selected', 'true'); - // E2E Normalized Interactivity leads the mode list for agentic. + // #736 made every latency mode a top-level tab and moved the default off E2E + // Normalized Interactivity, which still leads the strip without being selected. + cy.get('[data-testid="x-axis-mode-advanced"]').should('not.exist'); + for (const mode of ['e2e-normalized-interactivity', 'interactivity', 'e2e', 'ttft']) { + cy.get(`[data-testid="x-axis-mode-${mode}"]`).should('be.visible'); + } + cy.get('[data-testid="x-axis-mode-buttons"] [role="tab"]').should('have.length', 4); cy.get('[data-testid="x-axis-mode-buttons"] [role="tab"]') .first() - .should('have.attr', 'data-testid', 'x-axis-mode-e2e-normalized-interactivity'); - cy.get('[data-testid="chart-figure"] h2').should( - 'contain.text', - 'P90 E2E Normalized Interactivity', + .should('have.attr', 'data-testid', 'x-axis-mode-e2e-normalized-interactivity') + .and('have.attr', 'aria-selected', 'false'); + cy.get('[data-testid="x-axis-mode-interactivity"]').should( + 'have.attr', + 'aria-selected', + 'true', ); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); cy.get('[data-testid="chart-figure"] svg').should( 'contain.text', - 'P90 E2E Normalized Interactivity (tok/s/user)', + 'P90 Interactivity (tok/s/user)', ); }); - it('switches to Interactivity and updates the heading', () => { - selectAdvancedXAxisMode('interactivity', 'Interactivity'); + it('switches to E2E Normalized Interactivity and updates the heading', () => { + // The first entry into this mode fetches the trace-derived metrics, which the + // suite's intercept stubs; the default no longer fetches them on load. + cy.get('[data-testid="x-axis-mode-e2e-normalized-interactivity"]').click(); + cy.get('[data-testid="chart-figure"] h2').should( + 'contain.text', + 'P90 E2E Normalized Interactivity', + ); + selectXAxisMode('interactivity', 'Interactivity'); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); }); @@ -220,7 +234,7 @@ describe('X-Axis Mode Toggle (inference chart)', () => { it('shows the selected percentile in the Interactivity axis label', () => { // Explicitly select the mode — do not rely on the agentic default mode. - selectAdvancedXAxisMode('interactivity', 'Interactivity'); + selectXAxisMode('interactivity', 'Interactivity'); // Agentic plots percentile fields (p90_intvty), so the axis label carries it. cy.get('[data-testid="chart-figure"] svg').should( 'contain.text', @@ -252,12 +266,12 @@ describe('X-Axis Mode Toggle (inference chart)', () => { }); it('switches the x-axis to TTFT and updates the heading', () => { - selectAdvancedXAxisMode('ttft', 'TTFT'); + selectXAxisMode('ttft', 'TTFT'); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Time To First Token'); }); it('switches the x-axis to E2E Latency and updates the heading', () => { - selectAdvancedXAxisMode('e2e', 'E2E Latency'); + selectXAxisMode('e2e', 'E2E Latency'); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'End-to-end Latency'); cy.get('[data-testid="chart-figure"] svg').should('contain.text', 'P90 End-to-end Latency (s)'); }); @@ -298,19 +312,12 @@ describe('X-Axis Mode Toggle (inference chart)', () => { ); }); - // Documents the intended pairing of the Advanced menu with manual tab - // activation: focus may move to the lone remaining tab without the x-axis - // snapping back to it. The load-bearing guard for manual activation is the - // 8K/1K test below, where focus-activation is directly observable; this one - // states the agentic-side expectation. - it('keeps the Advanced selection when the remaining tab is focused', () => { - selectAdvancedXAxisMode('ttft', 'TTFT'); - - // Let the popover finish closing first: Radix restores focus to its own - // trigger on close, which would otherwise win the race against the focus - // below and mask the revert this test is guarding against. - cy.get('[data-testid="x-axis-mode-advanced-menu"]').should('not.exist'); - cy.get('[data-testid="x-axis-mode-advanced"]').should('have.focus'); + // Tabs are manually activated: moving focus along the strip must not change + // the x-axis. The load-bearing guard is the 8K/1K test below, where the same + // behaviour is asserted on the fixed-sequence strip; this one states the + // agentic-side expectation, across the agentic-only mode. + it('keeps the selected mode when another tab is focused', () => { + selectXAxisMode('ttft', 'TTFT'); cy.get('[data-testid="x-axis-mode-e2e-normalized-interactivity"]').focus(); cy.get('[data-testid="x-axis-mode-e2e-normalized-interactivity"]').should('have.focus'); @@ -320,14 +327,12 @@ describe('X-Axis Mode Toggle (inference chart)', () => { 'aria-selected', 'false', ); - cy.get('[data-testid="x-axis-mode-advanced"]') - .should('have.attr', 'data-state', 'active') - .and('contain.text', 'TTFT'); + cy.get('[data-testid="x-axis-mode-ttft"]').should('have.attr', 'aria-selected', 'true'); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Time To First Token'); }); it('switches back to Interactivity', () => { - selectAdvancedXAxisMode('interactivity', 'Interactivity'); + selectXAxisMode('interactivity', 'Interactivity'); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); cy.get('[data-testid="chart-figure"] svg').should( 'contain.text', @@ -338,7 +343,7 @@ describe('X-Axis Mode Toggle (inference chart)', () => { it('follows the percentile selector in the Interactivity axis label', () => { // Select p75 here rather than inheriting it from another test — the axis // label must track the selector on its own. - selectAdvancedXAxisMode('interactivity', 'Interactivity'); + selectXAxisMode('interactivity', 'Interactivity'); cy.get('[data-testid="percentile-selector"]').click(); cy.contains('[role="option"]', 'p75').click(); cy.get('[data-testid="chart-figure"] svg').should( @@ -352,24 +357,28 @@ describe('X-axis mode URL param', () => { // Regression: the reconcile effect used to run before availability resolved // the sequence. It recorded the fixed-seq placeholder kind, then treated the // switch to agentic as a user-driven kind change and clobbered the - // URL-restored mode with the agentic default (E2E Normalized Interactivity). + // URL-restored mode with the agentic default. + // + // The restored mode must differ from that default or the test cannot fail: + // #736 moved the default to Interactivity, so restoring Interactivity now + // proves nothing. TTFT is a mode the snap would visibly overwrite. it('keeps a URL-restored mode through the agentic sequence resolving', () => { interceptAgenticData(); interceptDerivedAgenticMetrics(); - cy.visit('/inference?i_seq=agentic-traces&i_xmode=interactivity', { + cy.visit('/inference?i_seq=agentic-traces&i_xmode=ttft', { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); unlockAgenticGate(win); }, }); cy.get('[data-testid="scenario-selector"]').should('contain.text', 'Agentic Traces'); - cy.get('[data-testid="x-axis-mode-advanced"]') - .should('have.attr', 'data-state', 'active') - .and('contain.text', 'Interactivity'); + cy.get('[data-testid="x-axis-mode-ttft"]') + .should('have.attr', 'aria-selected', 'true') + .and('contain.text', 'TTFT'); // Assert on the rendered chart too: the clobber happened one tick after // the buttons first painted, so a button-only check could pass too early. - cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); - cy.get('[data-testid="x-axis-mode-e2e-normalized-interactivity"]').should( + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Time To First Token'); + cy.get('[data-testid="x-axis-mode-interactivity"]').should( 'have.attr', 'aria-selected', 'false', @@ -454,8 +463,9 @@ describe('Label defaults for fixed-sequence scenarios', () => { cy.get('[data-testid="x-axis-mode-ttft"]').should('have.attr', 'aria-selected', 'true'); }); - // Only agentic nests the latency modes: E2E Normalized Interactivity is - // agentic-only, so collapsing them here would leave an empty tab strip. + // The strip is flat everywhere since #736; what is still specific to a fixed + // sequence is that E2E Normalized Interactivity is agentic-only, so this strip + // carries three tabs rather than four. it('keeps the flat x-axis strip with no Advanced menu', () => { interceptFixedSequenceData(); cy.visit('/inference?i_seq=8k%2F1k', { @@ -568,9 +578,19 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); }); + // Cypress clears intercepts between tests, so the derived-metrics stub is + // re-registered per test rather than once in `before`. Until #736 the + // agentic default was E2E Normalized Interactivity, so the fetch happened + // during `before` while the stub was still alive and React Query held the + // result for the rest of the spec. The default no longer fetches, so any + // later switch into that mode issues a fresh request. + beforeEach(() => { + interceptDerivedAgenticMetrics(); + }); + it('shows overlay (unofficial-run) watermark SVG when an overlay is loaded', () => { // Explicitly select Interactivity — do not rely on the agentic default mode. - selectAdvancedXAxisMode('interactivity', 'Interactivity'); + selectXAxisMode('interactivity', 'Interactivity'); // The unofficial-run pattern watermark appears when isUnofficialRun is true. cy.get('[data-testid="inference-chart-display"] svg pattern[id^="unofficial-pattern-"]').should( 'exist', @@ -586,7 +606,7 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () }); it('switches to ttft x-axis mode and renders SVG with overlay points', () => { - selectAdvancedXAxisMode('ttft', 'TTFT'); + selectXAxisMode('ttft', 'TTFT'); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Time To First Token'); // Overlay points render as triangles or circles inside the chart SVG. cy.get('[data-testid="inference-chart-display"] svg').should('exist'); diff --git a/packages/app/cypress/support/overlay-fixtures.ts b/packages/app/cypress/support/overlay-fixtures.ts index 2f326850..ffd00658 100644 --- a/packages/app/cypress/support/overlay-fixtures.ts +++ b/packages/app/cypress/support/overlay-fixtures.ts @@ -4,8 +4,11 @@ * The benchmark values are the real numbers from GitHub run 29682242847 * (GLM5.2 B300 agentic hicache, offload=on rows): * conc, p90_intvty (tok/s/user), tput_per_gpu, p90_e2el (s) - * C=4 is dominated on e2e by C=8 (12874 tok/s @ 33.1s vs 9415 @ 48.0s), which - * makes the set a ready-made probe for the e2e-restricted frontier behaviors. + * C=4 is dominated on e2e by C=8 (12874 tok/s @ 33.1s vs 9415 @ 48.0s). That + * used to make the set a probe for the e2e-restricted frontier; #736 removed + * that gating, so on the interactivity axes all five are Pareto-optimal — + * throughput falls monotonically as interactivity rises. Use + * `DOMINATED_CONFIG` when a test needs a point Optimal Only will actually drop. */ export const DEFAULT_MODEL_DB_KEY = 'dsv4'; export const AGENTIC_DATE = '2026-07-19'; @@ -39,9 +42,21 @@ export const metricsFor = (intvty: number, tput: number, e2el: number): Record 60) and throughput (12874 > 5000). Since #736 the + * Pareto frontier is computed from the selected axes alone, so this is the point + * Optimal Only must drop — the e2e-dominated C=4 above no longer is one. + */ +export const DOMINATED_CONFIG: [number, number, number, number] = [6, 60, 5000, 40]; + let idCursor = 900000; -export const b300Rows = (runUrl: string | null, hardware = 'b300') => - REAL_CONFIGS.map(([conc, intvty, tput, e2el]) => ({ +export const b300Rows = ( + runUrl: string | null, + hardware = 'b300', + configs: [number, number, number, number][] = REAL_CONFIGS, +) => + configs.map(([conc, intvty, tput, e2el]) => ({ id: runUrl ? 0 : idCursor++, hardware, framework: 'sglang', @@ -81,8 +96,18 @@ export const availability = [ }, ]; -/** Intercept availability + benchmarks + unofficial-run with the B300 fixture. */ -export const interceptOverlayRun = ({ overlayHardware = 'b300' } = {}) => { +/** + * Intercept availability + benchmarks + unofficial-run with the B300 fixture. + * `overlayConfigs` overrides only the overlay run's rows, leaving the official + * rows as the standard five. + */ +export const interceptOverlayRun = ({ + overlayHardware = 'b300', + overlayConfigs = REAL_CONFIGS, +}: { + overlayHardware?: string; + overlayConfigs?: [number, number, number, number][]; +} = {}) => { cy.intercept('GET', '/api/v1/availability', { body: availability }).as('availability'); cy.intercept('GET', '/api/v1/benchmarks*', { body: b300Rows(null) }).as('benchmarks'); cy.intercept('GET', '/api/unofficial-run*', { @@ -100,7 +125,7 @@ export const interceptOverlayRun = ({ overlayHardware = 'b300' } = {}) => { isNonMainBranch: true, }, ], - benchmarks: b300Rows(OVERLAY_RUN_URL, overlayHardware), + benchmarks: b300Rows(OVERLAY_RUN_URL, overlayHardware, overlayConfigs), evaluations: [], }, }).as('unofficialRun'); diff --git a/packages/app/src/components/calculator/useThroughputData.test.ts b/packages/app/src/components/calculator/useThroughputData.test.ts index eff30184..56664c33 100644 --- a/packages/app/src/components/calculator/useThroughputData.test.ts +++ b/packages/app/src/components/calculator/useThroughputData.test.ts @@ -1216,7 +1216,13 @@ describe('buildGpuGroups', () => { }); }); - it('restricts agentic interpolation to the same date-scoped e2e Pareto winners as the chart', () => { + it('keeps an e2e-dominated agentic point, because grouping no longer gates on the e2e frontier', () => { + // Until #736 this dropped every point that lost on E2E normalized + // interactivity, so grouping matched the chart's canonical frontier. That + // gating was removed deliberately: a frontier is now computed from the axes + // the reader actually selected, so grouping keeps every measured point and + // eligibility is decided downstream. Concurrency 2 is dominated on e2e by + // concurrency 1 and must survive anyway. const agenticRow = ( conc: number, interactivity: number, @@ -1245,7 +1251,6 @@ describe('buildGpuGroups', () => { agenticRow(1, 100, 20, 900), agenticRow(2, 80, 30, 800), // e2e-dominated by concurrency 1 agenticRow(4, 60, 40, 1200), - // A different date gets its own e2e frontier and must survive. agenticRow(8, 40, 50, 700, '2026-07-20'), ], { @@ -1257,7 +1262,7 @@ describe('buildGpuGroups', () => { ); const points = Object.values(grouped)[0]; - expect(points.map((point) => point.concurrency).toSorted()).toEqual([1, 4, 8]); + expect(points.map((point) => point.concurrency).toSorted()).toEqual([1, 2, 4, 8]); }); it('keeps agentic overlay frontiers isolated per unofficial run', () => { diff --git a/packages/app/src/components/inference/ui/ModelArchitectureDiagram.tsx b/packages/app/src/components/inference/ui/ModelArchitectureDiagram.tsx index 8cf2de0f..dc53b221 100644 --- a/packages/app/src/components/inference/ui/ModelArchitectureDiagram.tsx +++ b/packages/app/src/components/inference/ui/ModelArchitectureDiagram.tsx @@ -7,6 +7,8 @@ import Link from 'next/link'; import { useCallback, useEffect, useRef, useState } from 'react'; import { useTheme } from 'next-themes'; +import { getModelReleaseDate } from '@semianalysisai/inferencex-constants'; + import { Badge } from '@/components/ui/badge'; import type { Model } from '@/lib/data-mappings'; import { @@ -2266,6 +2268,9 @@ export default function ModelArchitectureDiagram({ const containerRef = useRef(null); const { resolvedTheme } = useTheme(); const arch = getModelArchitecture(model); + // Read from the one release-date table rather than a copy on `arch` — see the + // note in `ModelArchitecture`. `model` is the display name the table is keyed by. + const releaseDate = getModelReleaseDate(model); const toggleBlock = useCallback( (blockId: string) => { @@ -2441,10 +2446,10 @@ export default function ModelArchitectureDiagram({ )} - {arch.developer && arch.releaseDate && ( + {arch.developer && releaseDate && (

Released by {arch.developer} on{' '} - {new Date(arch.releaseDate).toLocaleDateString('en-US', { + {new Date(releaseDate).toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric', diff --git a/packages/app/src/lib/api-route-catalog.ts b/packages/app/src/lib/api-route-catalog.ts index f6305ee1..bba48de0 100644 --- a/packages/app/src/lib/api-route-catalog.ts +++ b/packages/app/src/lib/api-route-catalog.ts @@ -404,7 +404,10 @@ export const apiContractSourceDigests = [ }, { source: '../constants/src/models.ts', - sourceSha256: 'dca2b25f754adf90ac1bda31c5eb52503e3b3e8c72e684f10724c177c1317fac', + // Reviewed for the release-date table move: it adds MODEL_RELEASE_DATES and + // getModelReleaseDate. No published model name, alias, or parameter enum is + // touched, and no endpoint exposes a release date, so the docs stand. + sourceSha256: 'fde7ec056e048452663ddcaa46af137e9ec192676b94c701f57bcc4498417157', reviewArea: { en: 'Published benchmark and TCO model names, aliases, and parameter enums.', zh: '已发布基准与 TCO 模型名称、别名和参数枚举。', diff --git a/packages/app/src/lib/model-architectures.test.ts b/packages/app/src/lib/model-architectures.test.ts index efbf39e3..430147c7 100644 --- a/packages/app/src/lib/model-architectures.test.ts +++ b/packages/app/src/lib/model-architectures.test.ts @@ -1,3 +1,4 @@ +import { MODEL_RELEASE_DATES, getModelReleaseDate } from '@semianalysisai/inferencex-constants'; import { describe, expect, it } from 'vitest'; import { Model } from '@/lib/data-mappings'; @@ -20,6 +21,20 @@ import { MODEL_ARCHITECTURES, } from './model-architectures'; +describe('MODEL_RELEASE_DATES keys', () => { + it('names a real model, so no entry is dead weight', () => { + // The table is keyed by string because it lives in the constants package, + // below the `Model` enum. A key that matches no model never throws — the + // lookup returns null and every caller quietly falls back to its own + // default, so a typo here is invisible until someone notices the axis is + // anchored to the wrong date. This is the only place both are in scope. + const displayNames = new Set(Object.values(Model)); + for (const model of Object.keys(MODEL_RELEASE_DATES)) { + expect(displayNames.has(model), `${model} matches no Model display name`).toBe(true); + } + }); +}); + describe('MODEL_ARCHITECTURES', () => { it('has architecture data for all supported models', () => { const models = [ @@ -42,6 +57,29 @@ describe('MODEL_ARCHITECTURES', () => { } }); + it('carries no release date of its own, so there is nothing to diverge from', () => { + // This is the guard on the refactor, not a style rule. When each entry kept + // its own `releaseDate`, this file and MODEL_RELEASE_DATES disagreed about + // DeepSeek-V4-Pro — one of them had it releasing six weeks after its own + // first benchmark run — and both were rendered to users. Re-adding the field + // here recreates that, so the test fails on the field itself rather than + // waiting for the values to drift. + for (const [model, arch] of Object.entries(MODEL_ARCHITECTURES)) { + expect(arch, model).toBeDefined(); + expect(Object.keys(arch!), model).not.toContain('releaseDate'); + } + }); + + it('sources every architecture caption from the one release-date table', () => { + // The diagram prints "Released by {developer} on {date}", so an entry with a + // developer and no date silently drops the whole caption. Listing the models + // that have no sourced date keeps that visible instead of invisible. + const missing = Object.values(MODEL_ARCHITECTURES) + .filter((arch) => arch?.developer && !getModelReleaseDate(arch.model)) + .map((arch) => arch!.model); + expect(missing).toEqual([Model.MiniMax_M3]); + }); + it('ensures dense models have equal active and total params', () => { for (const arch of Object.values(MODEL_ARCHITECTURES)) { if (!arch) continue; @@ -143,7 +181,9 @@ describe('getModelArchitecture', () => { expect(arch?.numKVHeads).toBe(8); expect(arch?.ffnDim).toBe(28672); expect(arch?.vocabSize).toBe(128256); - expect(arch?.releaseDate).toBe('2024-07-23'); + // The release date is no longer carried here; it lives once in + // MODEL_RELEASE_DATES and is asserted in the constants package. + expect(getModelReleaseDate(Model.Llama3_1_70B)).toBe('2024-07-23'); }); it('returns architecture for DeepSeek R1 with MoE and MLA details', () => { diff --git a/packages/app/src/lib/model-architectures.ts b/packages/app/src/lib/model-architectures.ts index 43bd3022..f30393bb 100644 --- a/packages/app/src/lib/model-architectures.ts +++ b/packages/app/src/lib/model-architectures.ts @@ -90,8 +90,12 @@ export interface ModelArchitecture { contextWindow?: number; /** Special architectural features */ features?: string[]; - /** Release date (YYYY-MM-DD) */ - releaseDate?: string; + // No `releaseDate` here, deliberately. A model's release date is the same fact + // the Fleet Lifecycle time axis anchors on, so it lives once in + // `MODEL_RELEASE_DATES` (`@semianalysisai/inferencex-constants`) and is read + // through `getModelReleaseDate(arch.model)`. While this file kept its own copy + // the two disagreed: DeepSeek-V4-Pro was listed here as releasing six weeks + // after its own first benchmark run, and both values were rendered to users. /** Developer/Organization */ developer?: string; /** Link to model card or paper */ @@ -171,7 +175,6 @@ export const MODEL_ARCHITECTURES: Partial> = { 'Auxiliary-loss-free Load Balancing', 'Multi-Token Prediction', ], - releaseDate: '2025-05-28', developer: 'DeepSeek', sourceUrl: 'https://huggingface.co/deepseek-ai/DeepSeek-R1-0528', }, @@ -237,7 +240,6 @@ export const MODEL_ARCHITECTURES: Partial> = { 'FP4 Experts + FP8 Mixed Precision', 'Muon Optimizer', ], - releaseDate: '2026-06-08', developer: 'DeepSeek', sourceUrl: 'https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro', }, @@ -255,7 +257,6 @@ export const MODEL_ARCHITECTURES: Partial> = { ffnDim: 28672, contextWindow: 128000, features: ['Grouped Query Attention', 'RoPE'], - releaseDate: '2024-12-06', developer: 'Meta', sourceUrl: 'https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct', }, @@ -273,7 +274,6 @@ export const MODEL_ARCHITECTURES: Partial> = { ffnDim: 28672, contextWindow: 128000, features: ['Grouped Query Attention', 'RoPE'], - releaseDate: '2024-07-23', developer: 'Meta', sourceUrl: 'https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct', }, @@ -316,7 +316,6 @@ export const MODEL_ARCHITECTURES: Partial> = { 'YaRN RoPE (factor=32)', 'MXFP4 Quantization', ], - releaseDate: '2025-06-13', developer: 'OpenAI', sourceUrl: 'https://huggingface.co/openai/gpt-oss-120b', }, @@ -338,7 +337,6 @@ export const MODEL_ARCHITECTURES: Partial> = { denseFFNDim: 18432, contextWindow: 262144, features: ['Multi-head Latent Attention', 'DeepSeek-style MoE', 'YaRN RoPE'], - releaseDate: '2026-01-27', developer: 'Moonshot AI', sourceUrl: 'https://huggingface.co/moonshotai/Kimi-K2.5', }, @@ -407,7 +405,6 @@ export const MODEL_ARCHITECTURES: Partial> = { 'Native Multimodality (text/image/video)', 'MXFP4 Quantization', ], - releaseDate: '2026-06-13', developer: 'Moonshot AI', sourceUrl: 'https://huggingface.co/moonshotai/Kimi-K3', }, @@ -435,7 +432,6 @@ export const MODEL_ARCHITECTURES: Partial> = { 'Multi-Token Prediction (3 modules)', 'FP8 Quantization', ], - releaseDate: '2025-10-25', developer: 'MiniMax', sourceUrl: 'https://huggingface.co/MiniMaxAI/MiniMax-M2', }, diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index eae5bf74..62a86b87 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -1167,17 +1167,20 @@ describe('overview platform selection', () => { }); }); - it('restricts AgentX points to the E2E frontier on total throughput', () => { - // The slower-E2E point wins on output tokens but loses on total tokens, so - // the total-token frontier drops it and the tier read becomes unreachable. + it('reads AgentX tiers from every measured point, not just the E2E frontier', () => { + // The second point is dominated on total tokens (8100 < 9000) and slower on + // E2E (25 > 20). Until #736 the E2E frontier dropped it, leaving only the + // interactivity-40 point, which cannot reach the tier-50 read — so the tier + // came back null. That gating was removed deliberately, so the point now + // stands and the read lands on it exactly. const summary = buildOverviewModelSummary(Model.GLM_5_2, [ agenticRow(40, 20, 9000, 500, { hardware: 'b200', conc: 8 }), agenticRow(50, 25, 8100, 900, { hardware: 'b200', conc: 12 }), ]); const b200 = summary.platforms.find(({ hardware }) => hardware === 'b200')!; - expect(b200.read.value).toBeNull(); - expect(b200.missingReason).toBe('cannot_reach_at_tier'); + expect(b200.read.value).toBe(8100); + expect(b200.missingReason).toBeNull(); }); it('reports scenario-level missing coverage when AgentX rows lack usable P90 metrics', () => { diff --git a/packages/constants/src/models.test.ts b/packages/constants/src/models.test.ts index 34d50270..c251996a 100644 --- a/packages/constants/src/models.test.ts +++ b/packages/constants/src/models.test.ts @@ -2,6 +2,8 @@ import { describe, it, expect } from 'vitest'; import { DB_MODEL_TO_DISPLAY, DISPLAY_MODEL_TO_DB, + MODEL_RELEASE_DATES, + getModelReleaseDate, sequenceToIslOsl, islOslToSequence, } from './models'; @@ -36,6 +38,43 @@ describe('DB_MODEL_TO_DISPLAY / DISPLAY_MODEL_TO_DB consistency', () => { }); }); +describe('MODEL_RELEASE_DATES', () => { + it('stores every date as a real calendar day in YYYY-MM-DD', () => { + for (const [model, date] of Object.entries(MODEL_RELEASE_DATES)) { + expect(date, model).toMatch(/^\d{4}-\d{2}-\d{2}$/u); + // `new Date` happily accepts '2026-02-30' and rolls it into March, so + // round-tripping is the only way to catch a day that does not exist. + expect(new Date(`${date}T00:00:00Z`).toISOString().slice(0, 10), model).toBe(date); + } + }); + + it('has no model releasing in the future', () => { + // A future date is the signature of a typo'd year, and it would put the + // Fleet Lifecycle anchor after every sweep it is meant to precede. + const today = new Date().toISOString().slice(0, 10); + for (const [model, date] of Object.entries(MODEL_RELEASE_DATES)) { + expect( + date.localeCompare(today), + `${model} released ${date}, after today`, + ).toBeLessThanOrEqual(0); + } + }); + + it('is keyed by display name, never by DB model key', () => { + // Keying by 'dsv4' instead of 'DeepSeek-V4-Pro' does not fail loudly — the + // lookup just returns null forever and the caller silently falls back. + const dbKeys = new Set(Object.keys(DB_MODEL_TO_DISPLAY)); + for (const model of Object.keys(MODEL_RELEASE_DATES)) { + expect(dbKeys.has(model), `${model} is a DB key, not a display name`).toBe(false); + } + }); + + it('returns null rather than throwing for a model with no sourced date', () => { + expect(getModelReleaseDate('GLM-5.2')).toBeNull(); + expect(getModelReleaseDate('not-a-model')).toBeNull(); + }); +}); + describe('sequenceToIslOsl', () => { it('parses 1k/1k to 1024/1024', () => { expect(sequenceToIslOsl('1k/1k')).toEqual({ isl: 1024, osl: 1024 }); diff --git a/packages/constants/src/models.ts b/packages/constants/src/models.ts index 362de705..e3c126ea 100644 --- a/packages/constants/src/models.ts +++ b/packages/constants/src/models.ts @@ -77,3 +77,45 @@ export function rowToSequence(row: { if (row.isl === null || row.osl === null) return null; return islOslToSequence(row.isl, row.osl); } + +/** + * Model release dates, keyed by frontend display name (YYYY-MM-DD, UTC). + * + * **This is the only release-date table in the repo, and it must stay that way.** + * These dates were per-entry `releaseDate` fields on `MODEL_ARCHITECTURES`, which + * is the wrong home for them: a date that captions a diagram is the same fact as + * a date that anchors a revenue axis, and two copies of one fact drift. Callers + * go through `getModelReleaseDate`. + * + * **What the date means:** the day the weights became publicly downloadable — not + * the announcement, and not the day InferenceX started benchmarking the model. + * Inference optimisation cannot begin before the weights are out, so that + * publication is the honest zero for any "how far has this come since it + * shipped?" axis. Where a display name groups point releases (`GLM-5` covers + * both GLM-5 and GLM-5.1), the date is the *earliest* release in the bucket, + * because that is when the bucket's weights first existed. + * + * A date here must never postdate the model's first InferenceX sweep — a model + * cannot be benchmarked before it exists. That is the check that catches a wrong + * entry, and it is why these are worth sourcing individually rather than + * defaulting to whatever `MODELS.md` lists as the date the model was added. + * + * Only add an entry you can source, and cite the source in a comment. + * `getModelReleaseDate` returns null otherwise, so callers fall back to the + * earliest date they actually have data for. + */ +export const MODEL_RELEASE_DATES: Record = { + 'DeepSeek-R1-0528': '2025-05-28', + 'Llama-3.3-70B-Instruct-FP8': '2024-12-06', + 'Llama-3.1-70B-Instruct-FP8-KV': '2024-07-23', + 'gpt-oss-120b': '2025-06-13', + 'Kimi-K2.5': '2026-01-27', + 'Kimi-K3': '2026-06-13', + 'MiniMax-M2.5': '2025-10-25', + 'DeepSeek-V4-Pro': '2026-06-08', +}; + +/** Release date for a display model name, or null when we have no sourced date. */ +export function getModelReleaseDate(displayModel: string): string | null { + return MODEL_RELEASE_DATES[displayModel] ?? null; +}