Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions packages/app/cypress/e2e/csv-export-overlay.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,10 @@ function readCapturedCsv(): Cypress.Chainable<string> {
describe('Inference CSV export with an unofficial-run overlay', () => {
before(() => {
interceptOverlayRun();
// The agentic default mode is E2E Normalized Interactivity (which suppresses overlays and
// fetches derived metrics) — stub the fetch, then switch to the
// Interactivity mode this suite's overlay assertions rely on.
// Agentic charts default to Interactivity, where the overlay renders. The
// derived-metrics fetch now happens only under E2E Normalized Interactivity,
// so the stub below is a guard against a stray request rather than a
// dependency of this suite.
interceptDerivedAgenticMetrics();
cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, {
onBeforeLoad(win) {
Expand All @@ -47,12 +48,11 @@ describe('Inference CSV export with an unofficial-run overlay', () => {
},
});
cy.wait('@unofficialRun');
// Interactivity is nested under the Advanced menu on agentic charts.
cy.get('[data-testid="x-axis-mode-advanced"]').click();
cy.get('[data-testid="x-axis-mode-interactivity"]').click();
cy.get('[data-testid="x-axis-mode-advanced"]')
.should('have.attr', 'data-state', 'active')
.and('contain.text', 'Interactivity');
// Every x-axis metric is a top-level tab on agentic charts, and Interactivity
// is the default — clicked anyway so the suite does not depend on that.
cy.get('[data-testid="x-axis-mode-interactivity"]')
.click()
.should('have.attr', 'data-state', 'active');
cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should('exist');
});

Expand Down
33 changes: 24 additions & 9 deletions packages/app/cypress/e2e/overlay-legend-remove.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ describe('Official legend X works while an unofficial overlay is loaded', () =>
// Use distinct hardware so the engine-comparison exclusion policy does
// not resolve the official and unofficial rows as one competing family.
interceptOverlayRun({ overlayHardware: 'h100' });
// The agentic default mode is E2E Normalized Interactivity (which suppresses overlays and
// fetches derived metrics) — stub the fetch, then switch to Interactivity
// where the overlay renders.
// Agentic charts default to Interactivity, where the overlay renders. The
// derived-metrics fetch now happens only under E2E Normalized Interactivity,
// so the stub below is a guard against a stray request rather than a
// dependency of this suite.
interceptDerivedAgenticMetrics();
cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, {
onBeforeLoad(win) {
Expand All @@ -33,16 +34,28 @@ describe('Official legend X works while an unofficial overlay is loaded', () =>
},
});
cy.wait('@unofficialRun');
// Interactivity is nested under the Advanced menu on agentic charts.
cy.get('[data-testid="x-axis-mode-advanced"]').click();
cy.get('[data-testid="x-axis-mode-interactivity"]').click();
// Every x-axis metric is a top-level tab on agentic charts, and Interactivity
// is the default — clicked anyway so the suite does not depend on that.
cy.get('[data-testid="x-axis-mode-interactivity"]')
.click()
.should('have.attr', 'data-state', 'active');
cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1);
cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(
'have.length',
REAL_CONFIGS.length,
);
});

// Cypress clears intercepts between tests, so the derived-metrics stub is
// re-registered per test rather than once in `before`. Until #736 the
// agentic default was E2E Normalized Interactivity, so the fetch happened
// during `before` while the stub was still alive and React Query held the
// result for the rest of the spec. The default no longer fetches, so any
// later switch into that mode issues a fresh request.
beforeEach(() => {
interceptDerivedAgenticMetrics();
});

it('shows official points and an official legend entry initially', () => {
cy.get('[data-testid="inference-chart-display"] svg .dot-group').should(($dots) => {
expect(countVisible($dots), 'visible official points').to.be.greaterThan(0);
Expand All @@ -69,10 +82,12 @@ describe('Official legend X works while an unofficial overlay is loaded', () =>
cy.get('[data-testid="inference-chart-display"] svg .dot-group').should(($dots) => {
expect(countVisible($dots), 'visible official points after remove').to.eq(0);
});
// The overlay series is untouched. Optimal Only still hides its trace-less
// points because unofficial rows cannot join the canonical frontier.
// The overlay series is untouched — hiding an official SKU must not disturb
// it. All five overlay points stay visible under Optimal Only: they are
// non-dominated on the interactivity axes, and since #736 lacking persisted
// traces no longer excludes them from the frontier.
cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(($pts) => {
expect(countVisible($pts), 'visible overlay X markers').to.eq(0);
expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length);
});
// Inactive row: the hover affordance flips to the "+" restore indicator
// (explicit "clicking the name brings it back"), and the Hide X is gone.
Expand Down
64 changes: 40 additions & 24 deletions packages/app/cypress/e2e/overlay-optimal-only.cy.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,31 @@
/**
* Unofficial runs do not have persisted request traces, so they cannot join
* the canonical E2E Normalized Interactivity frontier. Optimal Only must hide
* them on every AgentX axis; Show All remains the explicit way to inspect them.
* Optimal Only must filter overlay (unofficial-run) points by the same rule it
* applies to official ones: the Pareto frontier of the *selected* axes.
*
* Until #736 the rule was different — unofficial runs have no persisted request
* traces, so they could never join the canonical E2E Normalized Interactivity
* frontier and Optimal Only hid every one of them. That gating is gone, so an
* overlay point now survives exactly when it is non-dominated on the axes on
* screen, and Show All remains the way to inspect the rest.
*/
import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e';
import {
countVisible,
DOMINATED_CONFIG,
interceptOverlayRun,
OVERLAY_RUN_ID,
REAL_CONFIGS,
} from '../support/overlay-fixtures';

describe('Overlay points follow canonical Optimal Only policy (agentic interactivity)', () => {
// The five real configs are all non-dominated on the interactivity axes, so a
// sixth, deliberately dominated point is what gives Optimal Only something to
// remove. Without it this suite would assert the same count either way and pass
// no matter what the filter did.
const OVERLAY_CONFIGS = [...REAL_CONFIGS, DOMINATED_CONFIG];

describe('Overlay points follow Optimal Only on the selected axes', () => {
before(() => {
interceptOverlayRun();
// The agentic default mode is E2E Normalized Interactivity (which suppresses overlays and
// fetches derived metrics) — stub the fetch, then switch to the
// Interactivity mode this suite is about.
interceptOverlayRun({ overlayConfigs: OVERLAY_CONFIGS });
interceptDerivedAgenticMetrics();
cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, {
onBeforeLoad(win) {
Expand All @@ -25,46 +34,53 @@ describe('Overlay points follow canonical Optimal Only policy (agentic interacti
},
});
cy.wait('@unofficialRun');
// Interactivity is nested under the Advanced menu on agentic charts.
cy.get('[data-testid="x-axis-mode-advanced"]').click();
cy.get('[data-testid="x-axis-mode-interactivity"]').click();
// Every x-axis metric is a top-level tab on agentic charts, and Interactivity
// is the default — clicked anyway so the suite does not depend on that.
cy.get('[data-testid="x-axis-mode-interactivity"]')
.click()
.should('have.attr', 'data-state', 'active');
cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1);
cy.get('[data-testid="x-axis-mode-advanced"]')
.should('have.attr', 'data-state', 'active')
.and('contain.text', 'Interactivity');
// All six are rendered; visibility is what Optimal Only changes.
cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(
'have.length',
REAL_CONFIGS.length,
OVERLAY_CONFIGS.length,
);
});

it('hides trace-less overlay points in the default Optimal Only view', () => {
// Cypress clears intercepts between tests, so the derived-metrics stub is
// re-registered per test rather than once in `before`. Until #736 the agentic
// default was E2E Normalized Interactivity, so the fetch happened during
// `before` while the stub was still alive and React Query held the result for
// the rest of the spec. The default no longer fetches.
beforeEach(() => {
interceptDerivedAgenticMetrics();
});

it('drops only the dominated overlay point in the default Optimal Only view', () => {
cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'checked');
// The deterministic derived-metric stub puts all five official rows on the
// canonical frontier.
cy.get('[data-testid="inference-chart-display"] svg .dot-group').should(($dots) => {
expect(countVisible($dots), 'visible official points').to.eq(REAL_CONFIGS.length);
});
// Overlay rows have no persisted trace ids and therefore no canonical
// frontier membership.
// Five of six survive: every real config is non-dominated on these axes, and
// the overlay's trace-less rows are no longer excluded for lacking traces.
cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(($pts) => {
expect(countVisible($pts), 'visible overlay X markers').to.eq(0);
expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length);
});
});

it('shows all overlay points when Optimal Only is turned off', () => {
cy.get('#scatter-hide-non-optimal').click();
cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'unchecked');
cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(($pts) => {
expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length);
expect(countVisible($pts), 'visible overlay X markers').to.eq(OVERLAY_CONFIGS.length);
});
});

it('re-hides trace-less overlay points when Optimal Only is re-enabled', () => {
it('re-drops the dominated overlay point when Optimal Only is re-enabled', () => {
cy.get('#scatter-hide-non-optimal').click();
cy.get('#scatter-hide-non-optimal').should('have.attr', 'data-state', 'checked');
cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should(($pts) => {
expect(countVisible($pts), 'visible overlay X markers').to.eq(0);
expect(countVisible($pts), 'visible overlay X markers').to.eq(REAL_CONFIGS.length);
});
});
});
Loading