Skip to content
Merged
30 changes: 30 additions & 0 deletions packages/app/cypress/component/inference-chart-controls.cy.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,36 @@ describe('Inference ChartControls', () => {
cy.get('@setSelectedYAxisMetric').should('have.been.calledOnce');
});

it('lists and selects the schema-v2 derived axes in the Measured Energy group', () => {
const options = [
{
key: 'y_measuredJPerSuccessfulQuery',
label: 'Measured Joules per Successful Query',
},
{
key: 'y_measuredWhPerSuccessfulQuery',
label: 'Measured Watt-hours per Successful Query',
},
{
key: 'y_measuredPowerPercentTdp',
label: 'Measured Average Power as Percent of TDP',
},
];

for (const option of options) {
cy.get('[data-testid="yaxis-metric-selector"]').click();
cy.contains('Measured Energy')
.parent()
.within(() => {
cy.contains('[role="option"]', option.label)
.scrollIntoView()
.should('be.visible')
.click();
});
cy.get('@setSelectedYAxisMetric').should('have.been.calledWith', option.key);
}
});

it('hides the GPU comparison section when no GPUs are selected', () => {
// Default mock: selectedGPUs = [] — GPU date range pickers should not render
cy.contains('Comparison Date Range').should('not.exist');
Expand Down
49 changes: 49 additions & 0 deletions packages/app/src/app/api/unofficial-run/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,55 @@ describe('normalizeArtifactRows', () => {
expect(m.mean_e2el).toBe(1.5);
});

it.each([
{
name: 'boolean verdict and junk-suffixed schema',
input: { power_valid: true, power_metric_schema_version: '2garbage' },
expectedVerdict: 0,
expectedSchema: undefined,
},
{
name: 'garbage verdict and valid numeric schema',
input: { power_valid: 'garbage', power_metric_schema_version: 2 },
expectedVerdict: 0,
expectedSchema: 2,
},
{
name: 'canonical numeric strings',
input: { power_valid: '1', power_metric_schema_version: '2' },
expectedVerdict: 1,
expectedSchema: 2,
},
{
name: 'explicit invalid verdict',
input: { power_valid: 0, power_metric_schema_version: 2 },
expectedVerdict: 0,
expectedSchema: 2,
},
{
name: 'legacy row without power contract fields',
input: {},
expectedVerdict: undefined,
expectedSchema: undefined,
},
])(
'normalizes overlay power contract discriminators: $name',
({ input, expectedVerdict, expectedSchema }) => {
const [row] = normalizeArtifactRows([rawRow(input)], '2026-03-01');

if (expectedVerdict === undefined) {
expect(row.metrics).not.toHaveProperty('power_valid');
} else {
expect(row.metrics.power_valid).toBe(expectedVerdict);
}
if (expectedSchema === undefined) {
expect(row.metrics).not.toHaveProperty('power_metric_schema_version');
} else {
expect(row.metrics.power_metric_schema_version).toBe(expectedSchema);
}
},
);

it('preserves recipe identity for unofficial overlays', () => {
const rows = normalizeArtifactRows(
[
Expand Down
15 changes: 15 additions & 0 deletions packages/app/src/components/ai-chart/types.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { describe, expect, it } from 'vitest';

import { validateSpec } from './types';

describe('validateSpec measured power axes', () => {
it.each([
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
'y_measuredPowerPercentTdp',
])('preserves %s as a benchmark Y-axis metric', (yAxisMetric) => {
const spec = validateSpec({ dataSource: 'benchmarks', yAxisMetric });

expect(spec.yAxisMetric).toBe(yAxisMetric);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,30 @@ describe('interpolateMetricAtInteractivity', () => {
expect(result!).toBeLessThan(3.5);
});

it.each([
'measuredJPerSuccessfulQuery',
'measuredWhPerSuccessfulQuery',
'measuredPowerPercentTdp',
] as const)('interpolates the derived measured metric %s', (metricKey) => {
const points = [
makePoint({
x: 20,
tpPerGpu: { y: 800, roof: false },
[metricKey]: { y: 80, roof: false },
}),
makePoint({
x: 60,
tpPerGpu: { y: 400, roof: false },
[metricKey]: { y: 40, roof: false },
}),
];

const result = interpolateMetricAtInteractivity(points, 40, metricKey);
expect(result).not.toBeNull();
expect(result!).toBeGreaterThan(40);
expect(result!).toBeLessThan(80);
});

it('returns null when metric field is missing from data points', () => {
const points = [
makePoint({ x: 20, tpPerGpu: { y: 800, roof: false } }),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
reciprocalMetricAt,
} from '@/components/calculator/useThroughputData';
import { useBenchmarkHistory } from '@/hooks/api/use-benchmark-history';
import { getHardwareKey } from '@/lib/chart-utils';
import { buildMeasuredPowerChartFields, getHardwareKey } from '@/lib/chart-utils';
import { getGpuSpecs, isKnownGpu } from '@/lib/constants';
import { rowToAggDataEntry } from '@/lib/benchmark-transform';
import type { BenchmarkRow } from '@/lib/api';
Expand Down Expand Up @@ -77,44 +77,7 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null {
jTotal: wrapMetric(power > 0 && tput ? (power * 1000) / tput : 0),
...(outputTput ? { jOutput: wrapMetric(power > 0 ? (power * 1000) / outputTput : 0) } : {}),
...(inputTput ? { jInput: wrapMetric(power > 0 ? (power * 1000) / inputTput : 0) } : {}),
...(typeof entry.avg_power_w === 'number'
? { measuredAvgPower: { y: entry.avg_power_w, roof: false } }
: {}),
...(typeof entry.joules_per_output_token === 'number'
? {
measuredJPerOutputToken: {
y: entry.joules_per_output_token,
roof: false,
},
}
: {}),
...(typeof entry.joules_per_total_token === 'number'
? {
measuredJPerTotalToken: {
y: entry.joules_per_total_token,
roof: false,
},
}
: {}),
...(typeof entry.prefill_avg_power_w === 'number'
? {
measuredPrefillAvgPower: {
y: entry.prefill_avg_power_w,
roof: false,
},
}
: {}),
...(typeof entry.decode_avg_power_w === 'number'
? { measuredDecodeAvgPower: { y: entry.decode_avg_power_w, roof: false } }
: {}),
...(typeof entry.joules_per_input_token === 'number'
? {
measuredJPerInputToken: {
y: entry.joules_per_input_token,
roof: false,
},
}
: {}),
...buildMeasuredPowerChartFields(entry, specs.tdp),
};
return point;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,23 @@
"y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)",
"y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗(含提示词)",
"y_measuredJPerTotalToken_roofline": "lower_right",
"y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y",
"y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)",
"y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(J/query)",
"y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query",
"y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗",
"y_measuredJPerSuccessfulQuery_roofline": "lower_right",
"y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y",
"y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)",
"y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(Wh/query)",
"y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query",
"y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗",
"y_measuredWhPerSuccessfulQuery_roofline": "lower_right",
"y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y",
"y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)",
"y_measuredPowerPercentTdp_labelZh": "实测平均功耗(TDP 占比)",
"y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP",
"y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比",
"y_cost_limit": 5,
"y_latency_limit": 60
},
Expand Down Expand Up @@ -331,6 +348,23 @@
"y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)",
"y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗(含提示词)",
"y_measuredJPerTotalToken_roofline": "lower_left",
"y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y",
"y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)",
"y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(J/query)",
"y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query",
"y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗",
"y_measuredJPerSuccessfulQuery_roofline": "lower_left",
"y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y",
"y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)",
"y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(Wh/query)",
"y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query",
"y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗",
"y_measuredWhPerSuccessfulQuery_roofline": "lower_left",
"y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y",
"y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)",
"y_measuredPowerPercentTdp_labelZh": "实测平均功耗(TDP 占比)",
"y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP",
"y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比",
"y_cost_limit": 5,
"y_latency_limit": 60
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ const MEASURED_POWER_METRICS = [
'y_measuredDecodeAvgPower',
] as const;

const QUERY_ENERGY_METRICS = [
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
] as const;

const defs = chartDefinitions as unknown as ChartDefinition[];
const interactivityDef = defs.find((d) => d.chartType === 'interactivity')!;
const e2eDef = defs.find((d) => d.chartType === 'e2e')!;
Expand Down Expand Up @@ -133,4 +138,27 @@ describe('measured-power Pareto direction', () => {
}
}
});

it.each(QUERY_ENERGY_METRICS)('%s is bilingual and lower-is-better', (metric) => {
expect(declaredDirection(interactivityDef, metric)).toBe('lower_right');
expect(declaredDirection(e2eDef, metric)).toBe('lower_left');
for (const chartDef of [interactivityDef, e2eDef]) {
expect(chartDef[metric]).toMatch(/\.y$/u);
expect(chartDef[`${metric}_label`]).toBeTruthy();
expect(chartDef[`${metric}_labelZh`]).toBeTruthy();
}
});

it('leaves %TDP without a Pareto direction on either block', () => {
// %TDP is a utilization gauge, not an efficiency frontier: a config running
// hotter is not "worse" along an axis the roofline can order, so declaring a
// corner would draw a frontier with no meaning. The axis still ships as a
// plottable, bilingual metric — it just never anchors a roofline.
for (const chartDef of [interactivityDef, e2eDef]) {
expect(chartDef.y_measuredPowerPercentTdp).toMatch(/\.y$/u);
expect(chartDef['y_measuredPowerPercentTdp_label']).toBeTruthy();
expect(chartDef['y_measuredPowerPercentTdp_labelZh']).toBeTruthy();
expect(declaredDirection(chartDef, 'y_measuredPowerPercentTdp')).toBeUndefined();
}
});
});
31 changes: 30 additions & 1 deletion packages/app/src/components/inference/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ export interface AggDataEntry {
power_valid?: number;
power_metric_schema_version?: number;
avg_power_w?: number;
joules_per_successful_query?: number;
joules_per_output_token?: number;
joules_per_total_token?: number;
// Multinode / disagg-only measured power. The aggregate_power.py runner
Expand Down Expand Up @@ -327,6 +328,9 @@ export interface InferenceData extends Partial<Omit<AggDataEntry, AggDataConflic
measuredJPerOutputToken?: { y: number; roof: boolean };
measuredJPerTotalToken?: { y: number; roof: boolean };
measuredJPerInputToken?: { y: number; roof: boolean };
measuredJPerSuccessfulQuery?: { y: number; roof: boolean };
measuredWhPerSuccessfulQuery?: { y: number; roof: boolean };
measuredPowerPercentTdp?: { y: number; roof: boolean };
}

/** Why a chart-ready point was intentionally excluded from the visible plot. */
Expand Down Expand Up @@ -370,7 +374,10 @@ export type YAxisMetricKey =
| 'measuredDecodeAvgPower'
| 'measuredJPerOutputToken'
| 'measuredJPerTotalToken'
| 'measuredJPerInputToken';
| 'measuredJPerInputToken'
| 'measuredJPerSuccessfulQuery'
| 'measuredWhPerSuccessfulQuery'
| 'measuredPowerPercentTdp';

/**
* Defines the configuration and labels for a specific chart.
Expand Down Expand Up @@ -500,6 +507,28 @@ export interface ChartDefinition {
y_measuredJPerTotalToken_label?: string;
y_measuredJPerTotalToken_title?: string;
y_measuredJPerTotalToken_roofline?: 'upper_right' | 'upper_left' | 'lower_left' | 'lower_right';
y_measuredJPerSuccessfulQuery?: string;
y_measuredJPerSuccessfulQuery_label?: string;
y_measuredJPerSuccessfulQuery_title?: string;
y_measuredJPerSuccessfulQuery_roofline?:
| 'upper_right'
| 'upper_left'
| 'lower_left'
| 'lower_right';
y_measuredWhPerSuccessfulQuery?: string;
y_measuredWhPerSuccessfulQuery_label?: string;
y_measuredWhPerSuccessfulQuery_title?: string;
y_measuredWhPerSuccessfulQuery_roofline?:
| 'upper_right'
| 'upper_left'
| 'lower_left'
| 'lower_right';
y_measuredPowerPercentTdp?: string;
y_measuredPowerPercentTdp_label?: string;
y_measuredPowerPercentTdp_title?: string;
// No `_roofline`: %TDP is a utilization gauge, not an efficiency frontier —
// there is no corner along which a config is "better", so the axis
// deliberately declares no Pareto direction.
y_cost_limit?: number;
y_latency_limit?: number;
}
Expand Down
3 changes: 3 additions & 0 deletions packages/app/src/components/inference/ui/ChartControls.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,9 @@ const METRIC_GROUPS: {
'y_measuredJPerInputToken',
'y_measuredJPerOutputToken',
'y_measuredJPerTotalToken',
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
'y_measuredPowerPercentTdp',
Comment thread
edwingao28 marked this conversation as resolved.
],
},
{ label: 'Custom User Values', labelZh: '自定义值', metrics: ['y_costUser', 'y_powerUser'] },
Expand Down
Loading