Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/app/cypress/component/scatter-graph.cy.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1369,6 +1369,7 @@ describe('ChartDisplay engine comparison guard', () => {
selectedSequence: Sequence.AgenticTraces,
selectedXAxisMode: 'interactivity' as const,
selectedXAxisMetric: 'p90_ttft',
bestPerSku: false,
activeHwTypes: new Set([officialKeys[0]]),
hwTypesWithData: new Set(officialKeys),
resolveComparisonSelection: resolveSelection,
Expand Down
2 changes: 2 additions & 0 deletions packages/app/cypress/support/mock-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,8 @@ export function createMockInferenceContext(
toggleHwType: namedStub('toggleHwType'),
removeHwType: namedStub('removeHwType'),
selectAllHwTypes: namedStub('selectAllHwTypes'),
bestPerSku: true,
setBestPerSku: namedStub('setBestPerSku'),
resolveComparisonSelection: (proposed) => ({
result: proposed,
keptGroup: null,
Expand Down
67 changes: 62 additions & 5 deletions packages/app/src/components/inference/InferenceContext.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ import {
comparisonExclusion as resolveComparisonExclusion,
} from './utils/comparison-exclusion';
import { resolveLabelState, serializeLabelState } from './utils/label-defaults';
import { bestSeriesPerSku } from './utils/best-series-per-sku';
import {
EMPTY_QUICK_FILTERS,
parseDeploymentModes,
Expand Down Expand Up @@ -327,6 +328,9 @@ export function InferenceProvider({
});

const [hideNonOptimal, setHideNonOptimal] = useState(() => getUrlParam('i_optimal') !== '0');
const [bestPerSku, setBestPerSku] = useState(
() => activeTab === 'inference' && getUrlParam('i_best') !== '0',
);
const labelScenarioKind = sequenceKind(effectiveSequence);
const initialLabelState = useMemo(
() =>
Expand Down Expand Up @@ -893,6 +897,37 @@ export function InferenceProvider({
extractHwKey,
);

const bestHwTypes = useMemo(() => {
const wantedType = selectedXAxisMode === 'interactivity' ? 'interactivity' : 'e2e';
const graph = graphs.find((candidate) => candidate.chartDefinition.chartType === wantedType);
if (!graph) return hwTypesWithData;
const direction =
graph.chartDefinition[
`${selectedYAxisMetric}_roofline` as keyof typeof graph.chartDefinition
];
if (
direction !== 'upper_right' &&
direction !== 'upper_left' &&
direction !== 'lower_left' &&
direction !== 'lower_right'
) {
return hwTypesWithData;
}
const best = bestSeriesPerSku(graph.data, direction);
return best.size > 0 ? best : hwTypesWithData;
Comment thread
adibarra marked this conversation as resolved.
}, [graphs, hwTypesWithData, selectedXAxisMode, selectedYAxisMetric]);

const setBestPerSkuAndApply = useCallback(
(enabled: boolean) => {
setBestPerSku(enabled);
const target = enabled ? bestHwTypes : hwTypesWithData;
setActiveHwTypes(resolveHwSelection(target).result);
setActivePresetId(null);
presetHwFilterRef.current = null;
},
[bestHwTypes, hwTypesWithData, resolveHwSelection, setActiveHwTypes],
);

// Direct fallback: apply pendingHwFilter when hwTypesWithData is already populated
// but useChartDataFilter didn't fire (e.g. re-selecting the same preset).
useEffect(() => {
Expand All @@ -911,6 +946,7 @@ export function InferenceProvider({
const next = toggleComparisonSelection(activeHwTypes, hw, hwTypesWithData);
if (!next) return;
setActiveHwTypes(next);
setBestPerSku(false);
setActivePresetId(null);
presetHwFilterRef.current = null;
},
Expand All @@ -920,6 +956,7 @@ export function InferenceProvider({
const removeHwType = useCallback(
(hw: string) => {
removeHwRaw(hw);
setBestPerSku(false);
setActivePresetId(null);
presetHwFilterRef.current = null;
},
Expand All @@ -941,6 +978,7 @@ export function InferenceProvider({
);
const removeActiveDate = useCallback((id: string) => removeDateRaw(id), [removeDateRaw]);
const selectAllHwTypes = useCallback(() => {
setBestPerSku(false);
if (exclusion) {
const { result, droppedGroups } = resolveHwSelection(hwTypesWithData, activeHwTypes);
setActiveHwTypes(result);
Expand Down Expand Up @@ -979,7 +1017,7 @@ export function InferenceProvider({
const precisionsKey = effectivePrecisions.join(',');
const hwResetKey = `${selectedModel}|${effectiveSequence}|${precisionsKey}|${
isUnofficialRun ? 'preview' : 'official'
}`;
}|${selectedYAxisMetric}|${selectedXAxisMode}`;
const lastHwResetKeyRef = useRef('');

// Restore legend-active selection from URL on first availability of
Expand Down Expand Up @@ -1052,23 +1090,26 @@ export function InferenceProvider({
// Scenarios that restrict standard-token engines (8K/1K, AgentX) keep one
// sticky group so their charts remain useful; variant-only rules retain
// the existing clear-all behavior.
const { result, droppedGroups } = resolveHwSelection(hwTypesWithData);
const automaticSelection = bestPerSku ? bestHwTypes : hwTypesWithData;
const { result, droppedGroups } = resolveHwSelection(automaticSelection);
setActiveHwTypes(result);
if (droppedGroups.length > 0) {
setEngineConflict({
kind: 'resolved',
...exclusionResolutionFamilies(hwTypesWithData, result, exclusion),
...exclusionResolutionFamilies(automaticSelection, result, exclusion),
});
}
return;
}
setActiveHwTypes(hwTypesWithData);
setActiveHwTypes(bestPerSku ? bestHwTypes : hwTypesWithData);
}, [
selectedModel,
effectiveSequence,
precisionsKey,
hwResetKey,
hwTypesWithData,
bestHwTypes,
bestPerSku,
exclusion,
pendingActiveHwTypes,
resolveHwSelection,
Expand Down Expand Up @@ -1177,6 +1218,16 @@ export function InferenceProvider({
// it equals the full set of items with data. Keeps share URLs short.
const iActiveStr = useMemo(() => {
if (activeHwTypes.size === 0) return '';
if (bestPerSku && activeHwTypes.size === bestHwTypes.size) {
let same = true;
for (const k of activeHwTypes) {
if (!bestHwTypes.has(k)) {
same = false;
break;
}
}
if (same) return '';
}
if (activeHwTypes.size === hwTypesWithData.size) {
let same = true;
for (const k of activeHwTypes) {
Expand All @@ -1188,7 +1239,7 @@ export function InferenceProvider({
if (same) return '';
}
return [...activeHwTypes].toSorted().join(',');
}, [activeHwTypes, hwTypesWithData]);
}, [activeHwTypes, hwTypesWithData, bestHwTypes, bestPerSku]);

const serializedLabelState = serializeLabelState(labelScenarioKind, {
showPointLabels,
Expand All @@ -1205,6 +1256,7 @@ export function InferenceProvider({
i_dstart: selectedDateRange.startDate,
i_dend: selectedDateRange.endDate,
i_optimal: hideNonOptimal ? '' : '0',
i_best: bestPerSku ? '' : '0',
i_label: serializedLabelState.i_label,
i_hc: highContrast ? '1' : '',
i_log: logScale ? '1' : '',
Expand Down Expand Up @@ -1234,6 +1286,7 @@ export function InferenceProvider({
selectedDates,
selectedDateRange,
hideNonOptimal,
bestPerSku,
showPointLabels,
highContrast,
logScale,
Expand Down Expand Up @@ -1374,6 +1427,8 @@ export function InferenceProvider({
toggleHwType,
removeHwType,
selectAllHwTypes,
bestPerSku,
setBestPerSku: setBestPerSkuAndApply,
resolveComparisonSelection: resolveHwSelection,
toggleComparisonSelection,
hardwareConfig,
Expand Down Expand Up @@ -1465,6 +1520,8 @@ export function InferenceProvider({
toggleHwType,
removeHwType,
selectAllHwTypes,
bestPerSku,
setBestPerSkuAndApply,
resolveHwSelection,
toggleComparisonSelection,

Expand Down
3 changes: 3 additions & 0 deletions packages/app/src/components/inference/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -743,6 +743,9 @@ export interface InferenceChartContextType {
toggleHwType: (hw: string) => void;
removeHwType: (hw: string) => void;
selectAllHwTypes: () => void;
/** Whether clean dashboard loads automatically keep the best configuration per physical SKU. */
bestPerSku: boolean;
setBestPerSku: (enabled: boolean) => void;
/** Resolve automatic official + `overlay:` hardware selections under the active scope rule. */
resolveComparisonSelection: (
proposed: Set<string>,
Expand Down
40 changes: 37 additions & 3 deletions packages/app/src/components/inference/ui/ChartDisplay.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
import { dataRunsForDate } from '@/components/inference/utils/runEnumeration';
import { matchesQuickFilters } from '@/components/inference/utils/quickFilters';
import { canonicalNormalizedFrontierIds } from '@/components/inference/utils/canonicalFrontier';
import { bestSeriesPerSku } from '@/components/inference/utils/best-series-per-sku';
import InferenceTable from '@/components/inference/ui/InferenceTable';
import ScatterGraph from '@/components/inference/ui/ScatterGraph';
import { Card } from '@/components/ui/card';
Expand Down Expand Up @@ -209,6 +210,7 @@ export default function ChartDisplay() {
selectedRunDate,
setIsLegendExpanded,
activeHwTypes,
bestPerSku,
activeDates,
selectedPercentile,
compareGpuPair,
Expand Down Expand Up @@ -446,6 +448,37 @@ export default function ChartDisplay() {
}
return eligibleKeys;
}, [graphs, selectedPrecisions, quickFilters]);
const scopedBestSelections = useMemo(() => {
if (!bestPerSku) return { official: officialScope, overlay: overlayScope };
const wantedType = selectedXAxisMode === 'interactivity' ? 'interactivity' : 'e2e';
const graph = graphs.find((candidate) => candidate.chartDefinition.chartType === wantedType);
const direction =
graph?.chartDefinition[`${selectedYAxisMetric}_roofline` as keyof ChartDefinition];
if (
!graph ||
(direction !== 'upper_right' &&
direction !== 'upper_left' &&
direction !== 'lower_left' &&
direction !== 'lower_right')
) {
return { official: officialScope, overlay: overlayScope };
}
const overlay = overlayDataByChartType[wantedType];
const officialBest = bestSeriesPerSku(graph.data, direction);
const overlayBest = bestSeriesPerSku(overlay?.data ?? [], direction);
return {
official: officialBest.size > 0 ? officialBest : officialScope,
overlay: overlayBest.size > 0 ? overlayBest : overlayScope,
};
}, [
bestPerSku,
graphs,
officialScope,
overlayDataByChartType,
overlayScope,
selectedXAxisMode,
selectedYAxisMetric,
]);
const overlayRowsScopeKey = `${selectedModel}|${selectedSequence}|${selectedPrecisions.join(
',',
)}|${unofficialRunInfos.map((run) => run.url).join(',')}`;
Expand All @@ -463,8 +496,8 @@ export default function ChartDisplay() {
const activeScopedOverlayKeys = new Set(
[...activeOverlayHwTypes].filter((key) => overlayScope.has(key)),
);
return overlayRowsScopeChanged ? overlayScope : activeScopedOverlayKeys;
}, [activeOverlayHwTypes, overlayScope, overlayRowsScopeChanged]);
return overlayRowsScopeChanged ? scopedBestSelections.overlay : activeScopedOverlayKeys;
}, [activeOverlayHwTypes, overlayScope, overlayRowsScopeChanged, scopedBestSelections.overlay]);
useEffect(() => {
const merged = new Set(activeOverlayHwTypes);
overlayScope.forEach((key) => merged.delete(key));
Expand All @@ -482,7 +515,7 @@ export default function ChartDisplay() {
// A scope change can render once before its official graphs arrive. Do not
// persist that transient empty set as an intentional legend selection.
if (overlayRowsScopeChanged && (!loading || officialScope.size > 0)) {
setLocalOfficialOverride(officialScope);
setLocalOfficialOverride(scopedBestSelections.official);
Comment thread
adibarra marked this conversation as resolved.
setAppliedOverlayRowsScopeKey(overlayRowsScopeKey);
}
}, [
Expand All @@ -491,6 +524,7 @@ export default function ChartDisplay() {
activeOverlayHwTypes,
loading,
officialScope,
scopedBestSelections.official,
overlayScope,
scopedActiveOverlayHwTypes,
setActiveOverlayHwTypes,
Expand Down
36 changes: 36 additions & 0 deletions packages/app/src/components/inference/ui/ScatterGraph.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ import {
renderKnownIssueAnnotations,
} from '@/components/inference/utils/knownIssueAnnotations';
import { matchesQuickFilters } from '@/components/inference/utils/quickFilters';
import { bestSeriesPerSku } from '@/components/inference/utils/best-series-per-sku';
import { changelogConfigToHwKey } from '@/components/inference/utils/changelogFormatters';
import {
buildFrontierContinuations,
Expand Down Expand Up @@ -393,6 +394,7 @@ const SCATTER_STRINGS = {
en: {
logScale: 'Log Scale',
optimalOnly: 'Optimal Only',
bestPerSku: 'Best per SKU',
optimalInfo:
'On agentic, optimal points must be Pareto-optimal on the selected x-axis and also belong to the E2E Normalized Interactivity frontier.',
labels: 'Labels',
Expand All @@ -408,6 +410,7 @@ const SCATTER_STRINGS = {
zh: {
logScale: '对数缩放',
optimalOnly: '仅最优',
bestPerSku: '每个 SKU 仅显示最佳配置',
optimalInfo:
'在智能体场景中,最优点既必须在当前横轴上满足 Pareto 最优,也必须属于端到端归一化交互性的 Pareto 前沿。',
labels: '标签',
Expand Down Expand Up @@ -443,6 +446,8 @@ const ScatterGraph = React.memo(
}: ScatterGraphProps) => {
const {
activeHwTypes,
bestPerSku,
setBestPerSku,
Comment thread
adibarra marked this conversation as resolved.
hardwareConfig: contextHardwareConfig,
toggleHwType,
removeHwType,
Expand Down Expand Up @@ -3259,6 +3264,37 @@ const ScatterGraph = React.memo(
track('latency_legend_expanded', { expanded });
}}
switches={[
{
id: 'scatter-best-per-sku',
label: legendT.bestPerSku,
checked: bestPerSku,
onCheckedChange: (checked: boolean) => {
setBestPerSku(checked);
if (overlayData) {
if (checked) {
const direction =
chartDefinition[
`${selectedYAxisMetric}_roofline` as keyof ChartDefinition
];
if (
direction === 'upper_right' ||
direction === 'upper_left' ||
direction === 'lower_left' ||
direction === 'lower_right'
) {
const selection = bestSeriesPerSku(data, direction);
for (const key of bestSeriesPerSku(overlayData.data, direction)) {
selection.add(`overlay:${key}`);
}
commitUnifiedSelection(selection);
}
} else {
resetUnifiedSelection();
}
}
track('inference_best_per_sku_toggled', { enabled: checked });
},
},
...(selectedYAxisMetric === 'y_inputTputPerGpu'
? []
: [
Expand Down
Loading