diff --git a/packages/app/cypress/component/inference-chart-controls.cy.tsx b/packages/app/cypress/component/inference-chart-controls.cy.tsx index f6db52302..6932cbe54 100644 --- a/packages/app/cypress/component/inference-chart-controls.cy.tsx +++ b/packages/app/cypress/component/inference-chart-controls.cy.tsx @@ -41,6 +41,36 @@ describe('Inference ChartControls', () => { cy.get('@setSelectedYAxisMetric').should('have.been.calledOnce'); }); + it('lists and selects the schema-v2 derived axes in the Measured Energy group', () => { + const options = [ + { + key: 'y_measuredJPerSuccessfulQuery', + label: 'Measured Joules per Successful Query', + }, + { + key: 'y_measuredWhPerSuccessfulQuery', + label: 'Measured Watt-hours per Successful Query', + }, + { + key: 'y_measuredPowerPercentTdp', + label: 'Measured Average Power as Percent of TDP', + }, + ]; + + for (const option of options) { + cy.get('[data-testid="yaxis-metric-selector"]').click(); + cy.contains('Measured Energy') + .parent() + .within(() => { + cy.contains('[role="option"]', option.label) + .scrollIntoView() + .should('be.visible') + .click(); + }); + cy.get('@setSelectedYAxisMetric').should('have.been.calledWith', option.key); + } + }); + it('hides the GPU comparison section when no GPUs are selected', () => { // Default mock: selectedGPUs = [] — GPU date range pickers should not render cy.contains('Comparison Date Range').should('not.exist'); diff --git a/packages/app/src/app/api/unofficial-run/route.test.ts b/packages/app/src/app/api/unofficial-run/route.test.ts index b9074a3ff..d4a8e5f4d 100644 --- a/packages/app/src/app/api/unofficial-run/route.test.ts +++ b/packages/app/src/app/api/unofficial-run/route.test.ts @@ -144,6 +144,55 @@ describe('normalizeArtifactRows', () => { expect(m.mean_e2el).toBe(1.5); }); + it.each([ + { + name: 'boolean verdict and junk-suffixed schema', + input: { power_valid: true, power_metric_schema_version: '2garbage' }, + expectedVerdict: 0, + expectedSchema: undefined, + }, + { + name: 'garbage verdict and valid numeric schema', + input: { power_valid: 'garbage', power_metric_schema_version: 2 }, + expectedVerdict: 0, + expectedSchema: 2, + }, + { + name: 'canonical numeric strings', + input: { power_valid: '1', power_metric_schema_version: '2' }, + expectedVerdict: 1, + expectedSchema: 2, + }, + { + name: 'explicit invalid verdict', + input: { power_valid: 0, power_metric_schema_version: 2 }, + expectedVerdict: 0, + expectedSchema: 2, + }, + { + name: 'legacy row without power contract fields', + input: {}, + expectedVerdict: undefined, + expectedSchema: undefined, + }, + ])( + 'normalizes overlay power contract discriminators: $name', + ({ input, expectedVerdict, expectedSchema }) => { + const [row] = normalizeArtifactRows([rawRow(input)], '2026-03-01'); + + if (expectedVerdict === undefined) { + expect(row.metrics).not.toHaveProperty('power_valid'); + } else { + expect(row.metrics.power_valid).toBe(expectedVerdict); + } + if (expectedSchema === undefined) { + expect(row.metrics).not.toHaveProperty('power_metric_schema_version'); + } else { + expect(row.metrics.power_metric_schema_version).toBe(expectedSchema); + } + }, + ); + it('preserves recipe identity for unofficial overlays', () => { const rows = normalizeArtifactRows( [ diff --git a/packages/app/src/components/ai-chart/types.test.ts b/packages/app/src/components/ai-chart/types.test.ts new file mode 100644 index 000000000..1837e1c04 --- /dev/null +++ b/packages/app/src/components/ai-chart/types.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest'; + +import { validateSpec } from './types'; + +describe('validateSpec measured power axes', () => { + it.each([ + 'y_measuredJPerSuccessfulQuery', + 'y_measuredWhPerSuccessfulQuery', + 'y_measuredPowerPercentTdp', + ])('preserves %s as a benchmark Y-axis metric', (yAxisMetric) => { + const spec = validateSpec({ dataSource: 'benchmarks', yAxisMetric }); + + expect(spec.yAxisMetric).toBe(yAxisMetric); + }); +}); diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts index a0e17a1dc..b718ca4e6 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts @@ -156,6 +156,30 @@ describe('interpolateMetricAtInteractivity', () => { expect(result!).toBeLessThan(3.5); }); + it.each([ + 'measuredJPerSuccessfulQuery', + 'measuredWhPerSuccessfulQuery', + 'measuredPowerPercentTdp', + ] as const)('interpolates the derived measured metric %s', (metricKey) => { + const points = [ + makePoint({ + x: 20, + tpPerGpu: { y: 800, roof: false }, + [metricKey]: { y: 80, roof: false }, + }), + makePoint({ + x: 60, + tpPerGpu: { y: 400, roof: false }, + [metricKey]: { y: 40, roof: false }, + }), + ]; + + const result = interpolateMetricAtInteractivity(points, 40, metricKey); + expect(result).not.toBeNull(); + expect(result!).toBeGreaterThan(40); + expect(result!).toBeLessThan(80); + }); + it('returns null when metric field is missing from data points', () => { const points = [ makePoint({ x: 20, tpPerGpu: { y: 800, roof: false } }), diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts index 1a300d998..16e8056da 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts @@ -11,7 +11,7 @@ import { reciprocalMetricAt, } from '@/components/calculator/useThroughputData'; import { useBenchmarkHistory } from '@/hooks/api/use-benchmark-history'; -import { getHardwareKey } from '@/lib/chart-utils'; +import { buildMeasuredPowerChartFields, getHardwareKey } from '@/lib/chart-utils'; import { getGpuSpecs, isKnownGpu } from '@/lib/constants'; import { rowToAggDataEntry } from '@/lib/benchmark-transform'; import type { BenchmarkRow } from '@/lib/api'; @@ -77,44 +77,7 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null { jTotal: wrapMetric(power > 0 && tput ? (power * 1000) / tput : 0), ...(outputTput ? { jOutput: wrapMetric(power > 0 ? (power * 1000) / outputTput : 0) } : {}), ...(inputTput ? { jInput: wrapMetric(power > 0 ? (power * 1000) / inputTput : 0) } : {}), - ...(typeof entry.avg_power_w === 'number' - ? { measuredAvgPower: { y: entry.avg_power_w, roof: false } } - : {}), - ...(typeof entry.joules_per_output_token === 'number' - ? { - measuredJPerOutputToken: { - y: entry.joules_per_output_token, - roof: false, - }, - } - : {}), - ...(typeof entry.joules_per_total_token === 'number' - ? { - measuredJPerTotalToken: { - y: entry.joules_per_total_token, - roof: false, - }, - } - : {}), - ...(typeof entry.prefill_avg_power_w === 'number' - ? { - measuredPrefillAvgPower: { - y: entry.prefill_avg_power_w, - roof: false, - }, - } - : {}), - ...(typeof entry.decode_avg_power_w === 'number' - ? { measuredDecodeAvgPower: { y: entry.decode_avg_power_w, roof: false } } - : {}), - ...(typeof entry.joules_per_input_token === 'number' - ? { - measuredJPerInputToken: { - y: entry.joules_per_input_token, - roof: false, - }, - } - : {}), + ...buildMeasuredPowerChartFields(entry, specs.tdp), }; return point; } diff --git a/packages/app/src/components/inference/inference-chart-config.json b/packages/app/src/components/inference/inference-chart-config.json index 392909349..5699acc16 100644 --- a/packages/app/src/components/inference/inference-chart-config.json +++ b/packages/app/src/components/inference/inference-chart-config.json @@ -164,6 +164,23 @@ "y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)", "y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗（含提示词）", "y_measuredJPerTotalToken_roofline": "lower_right", + "y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y", + "y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)", + "y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗（J/query）", + "y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query", + "y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗", + "y_measuredJPerSuccessfulQuery_roofline": "lower_right", + "y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y", + "y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)", + "y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗（Wh/query）", + "y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query", + "y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗", + "y_measuredWhPerSuccessfulQuery_roofline": "lower_right", + "y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y", + "y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)", + "y_measuredPowerPercentTdp_labelZh": "实测平均功耗（TDP 占比）", + "y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP", + "y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比", "y_cost_limit": 5, "y_latency_limit": 60 }, @@ -331,6 +348,23 @@ "y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)", "y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗（含提示词）", "y_measuredJPerTotalToken_roofline": "lower_left", + "y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y", + "y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)", + "y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗（J/query）", + "y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query", + "y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗", + "y_measuredJPerSuccessfulQuery_roofline": "lower_left", + "y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y", + "y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)", + "y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗（Wh/query）", + "y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query", + "y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗", + "y_measuredWhPerSuccessfulQuery_roofline": "lower_left", + "y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y", + "y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)", + "y_measuredPowerPercentTdp_labelZh": "实测平均功耗（TDP 占比）", + "y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP", + "y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比", "y_cost_limit": 5, "y_latency_limit": 60 } diff --git a/packages/app/src/components/inference/measured-power-direction.test.ts b/packages/app/src/components/inference/measured-power-direction.test.ts index ae307e431..391858907 100644 --- a/packages/app/src/components/inference/measured-power-direction.test.ts +++ b/packages/app/src/components/inference/measured-power-direction.test.ts @@ -30,6 +30,11 @@ const MEASURED_POWER_METRICS = [ 'y_measuredDecodeAvgPower', ] as const; +const QUERY_ENERGY_METRICS = [ + 'y_measuredJPerSuccessfulQuery', + 'y_measuredWhPerSuccessfulQuery', +] as const; + const defs = chartDefinitions as unknown as ChartDefinition[]; const interactivityDef = defs.find((d) => d.chartType === 'interactivity')!; const e2eDef = defs.find((d) => d.chartType === 'e2e')!; @@ -133,4 +138,27 @@ describe('measured-power Pareto direction', () => { } } }); + + it.each(QUERY_ENERGY_METRICS)('%s is bilingual and lower-is-better', (metric) => { + expect(declaredDirection(interactivityDef, metric)).toBe('lower_right'); + expect(declaredDirection(e2eDef, metric)).toBe('lower_left'); + for (const chartDef of [interactivityDef, e2eDef]) { + expect(chartDef[metric]).toMatch(/\.y$/u); + expect(chartDef[`${metric}_label`]).toBeTruthy(); + expect(chartDef[`${metric}_labelZh`]).toBeTruthy(); + } + }); + + it('leaves %TDP without a Pareto direction on either block', () => { + // %TDP is a utilization gauge, not an efficiency frontier: a config running + // hotter is not "worse" along an axis the roofline can order, so declaring a + // corner would draw a frontier with no meaning. The axis still ships as a + // plottable, bilingual metric — it just never anchors a roofline. + for (const chartDef of [interactivityDef, e2eDef]) { + expect(chartDef.y_measuredPowerPercentTdp).toMatch(/\.y$/u); + expect(chartDef['y_measuredPowerPercentTdp_label']).toBeTruthy(); + expect(chartDef['y_measuredPowerPercentTdp_labelZh']).toBeTruthy(); + expect(declaredDirection(chartDef, 'y_measuredPowerPercentTdp')).toBeUndefined(); + } + }); }); diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 444066bcb..d075f6bd2 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -142,6 +142,7 @@ export interface AggDataEntry { power_valid?: number; power_metric_schema_version?: number; avg_power_w?: number; + joules_per_successful_query?: number; joules_per_output_token?: number; joules_per_total_token?: number; // Multinode / disagg-only measured power. The aggregate_power.py runner @@ -327,6 +328,9 @@ export interface InferenceData extends Partial = {}): BenchmarkRow { }; } +function rawPowerArtifact(overrides: Record = {}): Record { + return { + infmax_model_prefix: 'dsr1', + hw: 'h200-nv', + framework: 'sglang', + precision: 'fp8', + isl: 1024, + osl: 1024, + conc: 128, + disagg: true, + prefill_tp: 8, + prefill_ep: 1, + prefill_num_workers: 1, + decode_tp: 8, + decode_ep: 1, + decode_num_workers: 1, + num_prefill_gpu: 8, + num_decode_gpu: 8, + median_e2el: 1.4, + median_intvty: 48, + tput_per_gpu: 100.5, + ...overrides, + }; +} + // --------------------------------------------------------------------------- // parseAvailableModelsAndSequences // --------------------------------------------------------------------------- @@ -276,3 +302,64 @@ describe('buildChartData', () => { }); }); }); + +describe('schema-v2 measured-power overlay data flow', () => { + it('normalizes an overlay artifact and exposes J/query, Wh/query, and percent TDP', () => { + const rows = normalizeArtifactRows( + [ + rawPowerArtifact({ + power_valid: '1', + power_metric_schema_version: '2', + avg_power_w: 560, + joules_per_successful_query: 1800, + }), + ], + '2026-08-12', + ); + const point = buildChartData(rows)['DeepSeek-R1-0528_1k/1k'].interactivity.data[0]; + + expect(rows[0].metrics.power_valid).toBe(1); + expect(rows[0].metrics.power_metric_schema_version).toBe(2); + expect(point.measuredJPerSuccessfulQuery?.y).toBe(1800); + expect(point.measuredWhPerSuccessfulQuery?.y).toBe(0.5); + expect(point.measuredPowerPercentTdp?.y).toBe(80); + }); + + it.each([ + { + name: 'malformed boolean verdict and junk schema', + contract: { power_valid: true, power_metric_schema_version: '2garbage' }, + }, + { + name: 'malformed string verdict', + contract: { power_valid: 'garbage', power_metric_schema_version: 2 }, + }, + { + name: 'explicit invalid verdict', + contract: { power_valid: 0, power_metric_schema_version: 2 }, + }, + ])('withholds measured values for $name', ({ contract }) => { + const rows = normalizeArtifactRows( + [ + rawPowerArtifact({ + ...contract, + avg_power_w: 560, + joules_per_successful_query: 1800, + avg_temp_c: 68.4, + workers: [{ role: 'agg', worker_idx: 0, num_gpus: 8, avg_power_w: 560 }], + }), + ], + '2026-08-12', + ); + const point = buildChartData(rows)['DeepSeek-R1-0528_1k/1k'].interactivity.data[0]; + + expect(rows[0].metrics.power_valid).toBe(0); + expect(point.avg_power_w).toBeUndefined(); + expect(point.joules_per_successful_query).toBeUndefined(); + expect(point.avg_temp_c).toBeUndefined(); + expect(point.workers).toBeUndefined(); + expect(point.measuredJPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredWhPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredPowerPercentTdp).toBeUndefined(); + }); +}); diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index 3c22c7386..fac8734d8 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -279,6 +279,20 @@ describe('rowToAggDataEntry', () => { expect(entry.joules_per_output_token).toBe(8.4); }); + it('passes through versioned whole-deployment joules per successful query', () => { + const entry = rowToAggDataEntry( + makeRow({ + metrics: { + power_valid: 1, + power_metric_schema_version: 2, + joules_per_successful_query: 1542.75, + }, + }), + ); + + expect(entry.joules_per_successful_query).toBe(1542.75); + }); + it('leaves measured power fields undefined for rows that predate the metric', () => { // Distinguishing "no measurement" from "0 W" matters: createChartDataPoint // uses typeof===number to decide whether to emit the measuredAvgPower field. @@ -320,6 +334,7 @@ describe('rowToAggDataEntry', () => { disagg: true, metrics: { avg_power_w: 650, + joules_per_successful_query: 1542.75, prefill_avg_power_w: 612.3, decode_avg_power_w: 701.5, joules_per_input_token: 1.2, @@ -333,6 +348,7 @@ describe('rowToAggDataEntry', () => { expect(entry.prefill_avg_power_w).toBe(612.3); expect(entry.decode_avg_power_w).toBe(701.5); expect(entry.joules_per_input_token).toBeUndefined(); + expect(entry.joules_per_successful_query).toBeUndefined(); expect(entry.joules_per_output_token).toBeUndefined(); expect(entry.joules_per_total_token).toBeUndefined(); }); @@ -347,6 +363,7 @@ describe('rowToAggDataEntry', () => { avg_power_w: 650, prefill_avg_power_w: 612.3, joules_per_input_token: 1.2, + joules_per_successful_query: 42.5, joules_per_output_token: 9.7, joules_per_total_token: 0.8, }, @@ -356,31 +373,37 @@ describe('rowToAggDataEntry', () => { expect(entry.avg_power_w).toBe(650); expect(entry.prefill_avg_power_w).toBe(612.3); expect(entry.joules_per_input_token).toBeUndefined(); + expect(entry.joules_per_successful_query).toBeUndefined(); expect(entry.joules_per_output_token).toBeUndefined(); expect(entry.joules_per_total_token).toBeUndefined(); }); - it('treats explicit power_valid=0 as authoritative and scrubs measured power', () => { + it('treats explicit power_valid=0 as authoritative and scrubs all measured telemetry', () => { const workers = [{ role: 'agg', worker_idx: 0, num_gpus: 8, avg_power_w: 685.5 }]; - const entry = rowToAggDataEntry( - makeRow({ - metrics: { - power_valid: 0, - power_metric_schema_version: 2, - avg_power_w: 685.5, - joules_per_input_token: 1.2, - joules_per_output_token: 8.4, - joules_per_total_token: 0.8, - prefill_avg_power_w: 612.3, - decode_avg_power_w: 701.5, - prefill_joules_per_input_token: 0.4, - decode_joules_per_output_token: 5.1, - }, - workers, - }), - ); + const row = makeRow({ + metrics: { + power_valid: 0, + power_metric_schema_version: 2, + avg_power_w: 685.5, + joules_per_successful_query: 1542.75, + joules_per_input_token: 1.2, + joules_per_output_token: 8.4, + joules_per_total_token: 0.8, + prefill_avg_power_w: 612.3, + decode_avg_power_w: 701.5, + prefill_joules_per_input_token: 0.4, + decode_joules_per_output_token: 5.1, + avg_temp_c: 68.4, + peak_temp_c: 79.2, + avg_util_pct: 88.5, + avg_mem_used_mb: 71234.5, + }, + workers, + }); + const entry = rowToAggDataEntry(row); expect(entry.avg_power_w).toBeUndefined(); + expect(entry.joules_per_successful_query).toBeUndefined(); expect(entry.joules_per_input_token).toBeUndefined(); expect(entry.joules_per_output_token).toBeUndefined(); expect(entry.joules_per_total_token).toBeUndefined(); @@ -388,7 +411,47 @@ describe('rowToAggDataEntry', () => { expect(entry.decode_avg_power_w).toBeUndefined(); expect(entry.prefill_joules_per_input_token).toBeUndefined(); expect(entry.decode_joules_per_output_token).toBeUndefined(); + expect(entry.avg_temp_c).toBeUndefined(); + expect(entry.peak_temp_c).toBeUndefined(); + expect(entry.avg_util_pct).toBeUndefined(); + expect(entry.avg_mem_used_mb).toBeUndefined(); expect(entry.workers).toBeUndefined(); + + const { chartData } = transformBenchmarkRows([row]); + const point = chartData.find((data) => data.length > 0)![0]; + expect(point.measuredJPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredWhPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredPowerPercentTdp).toBeUndefined(); + }); + + it('keeps measured telemetry compatible when a legacy row omits power_valid', () => { + const workers = [{ role: 'agg', worker_idx: 0, num_gpus: 8, avg_power_w: 560 }]; + const row = makeRow({ + metrics: { + avg_power_w: 560, + joules_per_successful_query: 1800, + avg_temp_c: 68.4, + peak_temp_c: 79.2, + avg_util_pct: 88.5, + avg_mem_used_mb: 71234.5, + }, + workers, + }); + const entry = rowToAggDataEntry(row); + + expect(entry.avg_power_w).toBe(560); + expect(entry.joules_per_successful_query).toBe(1800); + expect(entry.avg_temp_c).toBe(68.4); + expect(entry.peak_temp_c).toBe(79.2); + expect(entry.avg_util_pct).toBe(88.5); + expect(entry.avg_mem_used_mb).toBe(71234.5); + expect(entry.workers).toEqual(workers); + + const { chartData } = transformBenchmarkRows([row]); + const point = chartData.find((data) => data.length > 0)![0]; + expect(point.measuredJPerSuccessfulQuery?.y).toBe(1800); + expect(point.measuredWhPerSuccessfulQuery?.y).toBe(0.5); + expect(point.measuredPowerPercentTdp?.y).toBe(80); }); it('passes through per-worker measured power array intact', () => { diff --git a/packages/app/src/lib/benchmark-transform.ts b/packages/app/src/lib/benchmark-transform.ts index 120914f74..e7ba055cb 100644 --- a/packages/app/src/lib/benchmark-transform.ts +++ b/packages/app/src/lib/benchmark-transform.ts @@ -190,6 +190,10 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { power_valid: m.power_valid, power_metric_schema_version: m.power_metric_schema_version, avg_power_w: measuredPowerValid ? m.avg_power_w : undefined, + joules_per_successful_query: + measuredPowerValid && hasWholeDeploymentEnergySemantics + ? m.joules_per_successful_query + : undefined, joules_per_output_token: measuredPowerValid && hasWholeDeploymentEnergySemantics ? m.joules_per_output_token @@ -215,10 +219,10 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { // Cluster-wide GPU telemetry beyond power. Emitted when the perfmon CSVs // include the corresponding sample columns; left undefined otherwise so // the chart layer can distinguish "no measurement" from a real zero. - avg_temp_c: m.avg_temp_c, - peak_temp_c: m.peak_temp_c, - avg_util_pct: m.avg_util_pct, - avg_mem_used_mb: m.avg_mem_used_mb, + avg_temp_c: measuredPowerValid ? m.avg_temp_c : undefined, + peak_temp_c: measuredPowerValid ? m.peak_temp_c : undefined, + avg_util_pct: measuredPowerValid ? m.avg_util_pct : undefined, + avg_mem_used_mb: measuredPowerValid ? m.avg_mem_used_mb : undefined, // Per-worker measured power. Surfaced on BenchmarkRow as a sibling of the // scalar `metrics` dict (see api.ts). Narrow defensively so a malformed // payload can't poison downstream consumers. diff --git a/packages/app/src/lib/chart-utils.test.ts b/packages/app/src/lib/chart-utils.test.ts index b5d667cba..446a268f5 100644 --- a/packages/app/src/lib/chart-utils.test.ts +++ b/packages/app/src/lib/chart-utils.test.ts @@ -28,7 +28,7 @@ vi.mock('@/lib/constants', async (importOriginal) => { return { ...actual, getHardwareConfig: vi.fn(() => ({ label: 'H100', suffix: '' })), - getGpuSpecs: vi.fn(() => ({ power: 700, costh: 2.8, costn: 1.4, costr: 0.7 })), + getGpuSpecs: vi.fn(() => ({ power: 700, tdp: 700, costh: 2.8, costn: 1.4, costr: 0.7 })), }; }); @@ -849,6 +849,57 @@ describe('markRooflinePoints', () => { expect(mB.costh.roof).toBe(true); expect(mC.costh.roof).toBe(false); }); + + // Regression: the query-energy axes were registered in Y_AXIS_METRICS and had + // rooflines computed for them, but markRooflinePoints carried no branch, so + // every frontier point came back roof:false and the chart drew no roofline. + describe('query-energy axes', () => { + const chartDefQueryEnergy: ChartDefinition = { + ...chartDef, + y_measuredJPerSuccessfulQuery: 'measuredJPerSuccessfulQuery.y', + y_measuredJPerSuccessfulQuery_roofline: 'lower_right', + y_measuredWhPerSuccessfulQuery: 'measuredWhPerSuccessfulQuery.y', + y_measuredWhPerSuccessfulQuery_roofline: 'lower_right', + }; + + // lower_right (max x, min energy): A(x=1, 500 J) is the cheapest run and + // C(x=3, 900 J) is the fastest, so neither dominates the other and both sit + // on the front. B(x=2, 2400 J) is dominated by C on both axes. + const withEnergy = (x: number, joules: number, roof = false): InferenceData => ({ + ...pt(x, 0, 'h100', { tpPerGpuY: 50 }), + measuredJPerSuccessfulQuery: { y: joules, roof }, + measuredWhPerSuccessfulQuery: { y: joules / 3600, roof }, + }); + + const group = { h100: [withEnergy(1, 500), withEnergy(2, 2400), withEnergy(3, 900)] }; + + it('marks the frontier points on both query-energy axes', () => { + const rooflines = computeAllRooflines(group, chartDefQueryEnergy); + const marked = markRooflinePoints(group, rooflines, chartDefQueryEnergy); + + const a = marked.find((p) => p.x === 1)!; + const b = marked.find((p) => p.x === 2)!; + const c = marked.find((p) => p.x === 3)!; + + expect(a.measuredJPerSuccessfulQuery!.roof).toBe(true); + expect(c.measuredJPerSuccessfulQuery!.roof).toBe(true); + expect(b.measuredJPerSuccessfulQuery!.roof).toBe(false); + + expect(a.measuredWhPerSuccessfulQuery!.roof).toBe(true); + expect(c.measuredWhPerSuccessfulQuery!.roof).toBe(true); + expect(b.measuredWhPerSuccessfulQuery!.roof).toBe(false); + }); + + it('clears a stale roof flag on a dominated query-energy point', () => { + const dirty = { h100: [withEnergy(1, 500), withEnergy(2, 2400, true), withEnergy(3, 900)] }; + const rooflines = computeAllRooflines(dirty, chartDefQueryEnergy); + const marked = markRooflinePoints(dirty, rooflines, chartDefQueryEnergy); + + const b = marked.find((p) => p.x === 2)!; + expect(b.measuredJPerSuccessfulQuery!.roof).toBe(false); + expect(b.measuredWhPerSuccessfulQuery!.roof).toBe(false); + }); + }); }); // --------------------------------------------------------------------------- @@ -1333,6 +1384,29 @@ describe('createChartDataPoint measured power fields', () => { expect(point.measuredJPerOutputToken!.y).toBe(8.4); }); + it('derives J/query, Wh/query, and percent TDP from validated source fields', () => { + const e = entry({ avg_power_w: 560, joules_per_successful_query: 1800 }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + + expect(point.measuredJPerSuccessfulQuery?.y).toBe(1800); + expect(point.measuredWhPerSuccessfulQuery?.y).toBe(0.5); + expect(point.measuredPowerPercentTdp?.y).toBe(80); + }); + + it('omits derived query and TDP axes when their inputs are absent', () => { + const point = createChartDataPoint( + '2025-01-01', + entry(), + 'median_e2el', + 'tput_per_gpu', + 'h100', + ); + + expect(point.measuredJPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredWhPerSuccessfulQuery).toBeUndefined(); + expect(point.measuredPowerPercentTdp).toBeUndefined(); + }); + it('omits both fields when neither is on the entry', () => { // Legacy runs predating aggregate_power.py. const point = createChartDataPoint( diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts index 087b3a114..397de45cb 100644 --- a/packages/app/src/lib/chart-utils.ts +++ b/packages/app/src/lib/chart-utils.ts @@ -191,6 +191,9 @@ export const Y_AXIS_METRICS = [ 'y_measuredJPerOutputToken', 'y_measuredJPerTotalToken', 'y_measuredJPerInputToken', + 'y_measuredJPerSuccessfulQuery', + 'y_measuredWhPerSuccessfulQuery', + 'y_measuredPowerPercentTdp', ] as const; export type YAxisMetric = (typeof Y_AXIS_METRICS)[number]; @@ -451,26 +454,59 @@ export function createChartDataPoint( } : {}), - // Measured power / energy from runner's aggregate_power.py. Gated on the - // raw fields existing so points from runs predating the measurement land - // without these keys and the chart correctly filters them out. + ...buildMeasuredPowerChartFields(entry, specs.tdp), + }; +} + +type MeasuredPowerChartFields = Partial< + Pick< + InferenceData, + | 'measuredAvgPower' + | 'measuredPrefillAvgPower' + | 'measuredDecodeAvgPower' + | 'measuredJPerOutputToken' + | 'measuredJPerTotalToken' + | 'measuredJPerInputToken' + | 'measuredJPerSuccessfulQuery' + | 'measuredWhPerSuccessfulQuery' + | 'measuredPowerPercentTdp' + > +>; + +const measuredMetric = (y: number): { y: number; roof: boolean } => ({ y, roof: false }); + +/** Build the measured fields shared by scatter points and historical trends. */ +export function buildMeasuredPowerChartFields( + entry: AggDataEntry, + tdpWatts: number, +): MeasuredPowerChartFields { + return { ...(typeof entry.avg_power_w === 'number' - ? { measuredAvgPower: { y: entry.avg_power_w, roof: false } } + ? { measuredAvgPower: measuredMetric(entry.avg_power_w) } : {}), ...(typeof entry.prefill_avg_power_w === 'number' - ? { measuredPrefillAvgPower: { y: entry.prefill_avg_power_w, roof: false } } + ? { measuredPrefillAvgPower: measuredMetric(entry.prefill_avg_power_w) } : {}), ...(typeof entry.decode_avg_power_w === 'number' - ? { measuredDecodeAvgPower: { y: entry.decode_avg_power_w, roof: false } } + ? { measuredDecodeAvgPower: measuredMetric(entry.decode_avg_power_w) } : {}), ...(typeof entry.joules_per_output_token === 'number' - ? { measuredJPerOutputToken: { y: entry.joules_per_output_token, roof: false } } + ? { measuredJPerOutputToken: measuredMetric(entry.joules_per_output_token) } : {}), ...(typeof entry.joules_per_total_token === 'number' - ? { measuredJPerTotalToken: { y: entry.joules_per_total_token, roof: false } } + ? { measuredJPerTotalToken: measuredMetric(entry.joules_per_total_token) } : {}), ...(typeof entry.joules_per_input_token === 'number' - ? { measuredJPerInputToken: { y: entry.joules_per_input_token, roof: false } } + ? { measuredJPerInputToken: measuredMetric(entry.joules_per_input_token) } + : {}), + ...(typeof entry.joules_per_successful_query === 'number' + ? { + measuredJPerSuccessfulQuery: measuredMetric(entry.joules_per_successful_query), + measuredWhPerSuccessfulQuery: measuredMetric(entry.joules_per_successful_query / 3600), + } + : {}), + ...(typeof entry.avg_power_w === 'number' && tdpWatts > 0 + ? { measuredPowerPercentTdp: measuredMetric((entry.avg_power_w / tdpWatts) * 100) } : {}), }; } @@ -663,7 +699,9 @@ export const calculateRoofline = ( | `measuredDecodeAvgPower.y` | `measuredJPerOutputToken.y` | `measuredJPerTotalToken.y` - | `measuredJPerInputToken.y`, + | `measuredJPerInputToken.y` + | `measuredJPerSuccessfulQuery.y` + | `measuredWhPerSuccessfulQuery.y`, rooflineDirection: 'upper_right' | 'upper_left' | 'lower_left' | 'lower_right', ): InferenceData[] => { // Exclude degenerate x <= 0 points (see isFrontierEligible) so they never @@ -741,7 +779,9 @@ export function computeAllRooflines( | `measuredDecodeAvgPower.y` | `measuredJPerOutputToken.y` | `measuredJPerTotalToken.y` - | `measuredJPerInputToken.y`, + | `measuredJPerInputToken.y` + | `measuredJPerSuccessfulQuery.y` + | `measuredWhPerSuccessfulQuery.y`, rooflineDirection, ); } @@ -791,6 +831,8 @@ export function markRooflinePoints( if (newPoint.measuredJPerOutputToken) newPoint.measuredJPerOutputToken.roof = false; if (newPoint.measuredJPerTotalToken) newPoint.measuredJPerTotalToken.roof = false; if (newPoint.measuredJPerInputToken) newPoint.measuredJPerInputToken.roof = false; + if (newPoint.measuredJPerSuccessfulQuery) newPoint.measuredJPerSuccessfulQuery.roof = false; + if (newPoint.measuredWhPerSuccessfulQuery) newPoint.measuredWhPerSuccessfulQuery.roof = false; for (const chartDefYKey of Y_AXIS_METRICS) { const rooflinePoints = computedRooflines[hwKey]?.[chartDefYKey]; @@ -868,6 +910,16 @@ export function markRooflinePoints( newPoint.measuredJPerTotalToken.roof = onCurrentRoofline; } else if (chartDefYKey === 'y_measuredJPerInputToken' && newPoint.measuredJPerInputToken) { newPoint.measuredJPerInputToken.roof = onCurrentRoofline; + } else if ( + chartDefYKey === 'y_measuredJPerSuccessfulQuery' && + newPoint.measuredJPerSuccessfulQuery + ) { + newPoint.measuredJPerSuccessfulQuery.roof = onCurrentRoofline; + } else if ( + chartDefYKey === 'y_measuredWhPerSuccessfulQuery' && + newPoint.measuredWhPerSuccessfulQuery + ) { + newPoint.measuredWhPerSuccessfulQuery.roof = onCurrentRoofline; } } finalProcessedData.push(newPoint); diff --git a/packages/app/src/lib/constants.test.ts b/packages/app/src/lib/constants.test.ts index f561591b6..14ec264f2 100644 --- a/packages/app/src/lib/constants.test.ts +++ b/packages/app/src/lib/constants.test.ts @@ -224,6 +224,7 @@ describe('getHardwareConfig', () => { describe('getGpuSpecs', () => { it('returns specs for a base GPU key', () => { const specs = getGpuSpecs('h100'); + expect(specs.tdp).toBe(700); expect(specs.power).toBe(1.37); expect(specs.costh).toBe(1.17); expect(specs.costn).toBe(1.55); @@ -244,6 +245,7 @@ describe('getGpuSpecs', () => { it('returns zero specs for unknown GPU', () => { const specs = getGpuSpecs('nonexistent'); expect(specs.power).toBe(0); + expect(specs.tdp).toBe(0); expect(specs.costh).toBe(0); expect(specs.costn).toBe(0); expect(specs.costr).toBe(0); @@ -253,6 +255,7 @@ describe('getGpuSpecs', () => { for (const [base, entry] of Object.entries(HW_REGISTRY)) { const result = getGpuSpecs(base); expect(result.power).toBe(entry.power); + expect(result.tdp).toBe(entry.tdp); expect(result.costh).toBe(entry.costh); } }); diff --git a/packages/app/src/lib/constants.ts b/packages/app/src/lib/constants.ts index a720077d9..b0a8ebe66 100644 --- a/packages/app/src/lib/constants.ts +++ b/packages/app/src/lib/constants.ts @@ -15,13 +15,14 @@ export const TABLEAU_10 = [ ] as const; export interface GpuSpecs { + tdp: number; power: number; costh: number; costn: number; costr: number; } -const DEFAULT_SPECS: GpuSpecs = { power: 0, costh: 0, costn: 0, costr: 0 }; +const DEFAULT_SPECS: GpuSpecs = { tdp: 0, power: 0, costh: 0, costn: 0, costr: 0 }; /** * Look up power/cost specs for a hardware key by extracting the base GPU name. @@ -31,7 +32,13 @@ export function getGpuSpecs(hwKey: string): GpuSpecs { const base = hwKey.split(/[-_]/u)[0]; const entry = HW_REGISTRY[base]; if (!entry) return DEFAULT_SPECS; - return { power: entry.power, costh: entry.costh, costn: entry.costn, costr: entry.costr }; + return { + tdp: entry.tdp, + power: entry.power, + costh: entry.costh, + costn: entry.costn, + costr: entry.costr, + }; } /** Build the vendor prefix string for the `gpu` tooltip field. */ diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 8a0914601..370ea7f50 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -135,6 +135,7 @@ export const METRIC_KEYS = new Set([ // power_metric_schema_version: version 2 defines every unprefixed // joules_per_* field as whole-deployment energy // avg_power_w: mean per-GPU draw (W) during the load window + // joules_per_successful_query: whole-deployment energy / successful requests // joules_per_output_token: energy / total_output_tokens. CLUSTER-WIDE on // schema-version-2 rows, including disaggregated runs. // joules_per_total_token: total_system_energy / (total_input + total_output) @@ -143,6 +144,7 @@ export const METRIC_KEYS = new Set([ 'power_valid', 'power_metric_schema_version', 'avg_power_w', + 'joules_per_successful_query', 'joules_per_output_token', 'joules_per_total_token', // multinode / disagg role splits (emitted only when the deployment has