From 38b28e30d180b3b77e0480fd386b727522d2622c Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 5 Jul 2026 02:22:15 +0200 Subject: [PATCH 1/3] Use stable hashed row keys for msbench eval sync --- .../src/functions/syncMsbenchEvalMetrics.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts b/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts index f07209b01..1c077cd51 100644 --- a/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts +++ b/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts @@ -1,6 +1,7 @@ import { app, HttpRequest, HttpResponseInit, InvocationContext, Timer } from "@azure/functions"; import { TableClient } from "@azure/data-tables"; import { AzureCliCredential, ManagedIdentityCredential } from "@azure/identity"; +import { createHash } from "node:crypto"; import { listMsbenchDates, enumerateMsbenchBlobs, getMsbenchBlobContent } from "../msbenchBlobEnumerator"; import type { BlobTreeNode } from "../shared/blobTree"; @@ -13,10 +14,20 @@ interface EvalReport { total_steps: number; model: string; instance_id: string; + benchmark?: string; resolved?: boolean; [key: string]: unknown; } +function toSafeString(value: string | undefined): string { + return value?.trim() ? value : "unknown"; +} + +function buildMsbenchRowKey(instanceId: string, benchmark: string, model: string): string { + const payload = `${instanceId}|${benchmark}|${model}`; + return createHash("sha256").update(payload).digest("base64url"); +} + function getEvalTableClient(): TableClient { if (!MSBENCH_STORAGE_ACCOUNT) { throw new Error("MSBENCH_STORAGE_ACCOUNT environment variable is not set"); @@ -106,12 +117,14 @@ async function runSync(context: InvocationContext, force = false): Promise<{ syn for (const report of reports) { if (!report) continue; - const benchmark = report.instance_id || "unknown"; - const model = report.model || "unknown"; + const instanceId = toSafeString(report.instance_id); + const benchmark = toSafeString(report.benchmark ?? instanceId); + const model = toSafeString(report.model); await tableClient.upsertEntity({ partitionKey: date, - rowKey: `${benchmark}_${model}`, + rowKey: buildMsbenchRowKey(instanceId, benchmark, model), + instance_id: instanceId, benchmark, model, totalConsumedTokens: Number(report.total_consumed_tokens) || 0, From 6d6e1f5c1a9e02c8ea7393df4f3f60f0c4e2a727 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 5 Jul 2026 02:33:18 +0200 Subject: [PATCH 2/3] Tighten msbench row key hashing and avoid force duplicates --- .../src/functions/syncMsbenchEvalMetrics.ts | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts b/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts index 1c077cd51..0f0a4691e 100644 --- a/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts +++ b/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts @@ -20,11 +20,12 @@ interface EvalReport { } function toSafeString(value: string | undefined): string { - return value?.trim() ? value : "unknown"; + const trimmed = value?.trim(); + return trimmed ? trimmed : "unknown"; } function buildMsbenchRowKey(instanceId: string, benchmark: string, model: string): string { - const payload = `${instanceId}|${benchmark}|${model}`; + const payload = JSON.stringify([instanceId, benchmark, model]); return createHash("sha256").update(payload).digest("base64url"); } @@ -68,6 +69,22 @@ async function getProcessedDates(tableClient: TableClient): Promise> return dates; } +async function deleteDatePartition(tableClient: TableClient, partitionKey: string, context: InvocationContext): Promise { + let deleted = 0; + for await (const entity of tableClient.listEntities({ + queryOptions: { + filter: `PartitionKey eq '${partitionKey}'`, + select: ["rowKey"], + }, + })) { + await tableClient.deleteEntity(partitionKey, entity.rowKey); + deleted++; + } + + context.log(`Deleted ${deleted} existing rows for date ${partitionKey} before re-sync`); + return deleted; +} + async function runSync(context: InvocationContext, force = false): Promise<{ synced: number; dates?: string[]; message?: string }> { const tableClient = getEvalTableClient(); const blobDates = await listMsbenchDates(); @@ -95,6 +112,10 @@ async function runSync(context: InvocationContext, force = false): Promise<{ syn const dateNode = tree[date]; if (!dateNode) continue; + if (force) { + await deleteDatePartition(tableClient, date, context); + } + const evalPaths = collectEvalReportPaths(dateNode); // Download reports with bounded concurrency From 11649a92096607aa20c016f162369db6d1b217de Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 5 Jul 2026 02:36:58 +0200 Subject: [PATCH 3/3] Use path fallback for missing msbench identity fields --- .../src/functions/syncMsbenchEvalMetrics.ts | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts b/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts index 0f0a4691e..9b91bb725 100644 --- a/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts +++ b/dashboard/sync/src/functions/syncMsbenchEvalMetrics.ts @@ -19,9 +19,13 @@ interface EvalReport { [key: string]: unknown; } -function toSafeString(value: string | undefined): string { +interface EvalReportWithPath extends EvalReport { + blobPath: string; +} + +function toSafeString(value: string | undefined): string | undefined { const trimmed = value?.trim(); - return trimmed ? trimmed : "unknown"; + return trimmed ? trimmed : undefined; } function buildMsbenchRowKey(instanceId: string, benchmark: string, model: string): string { @@ -119,14 +123,15 @@ async function runSync(context: InvocationContext, force = false): Promise<{ syn const evalPaths = collectEvalReportPaths(dateNode); // Download reports with bounded concurrency - const reports: (EvalReport | null)[] = []; + const reports: (EvalReportWithPath | null)[] = []; for (let i = 0; i < evalPaths.length; i += CONCURRENCY_LIMIT) { const batch = evalPaths.slice(i, i + CONCURRENCY_LIMIT); const batchResults = await Promise.all( batch.map(async (path) => { try { const raw = await getMsbenchBlobContent(path); - return JSON.parse(raw) as EvalReport; + const report = JSON.parse(raw) as EvalReport; + return { ...report, blobPath: path }; } catch { context.log(`Skipping malformed eval report: ${path}`); return null; @@ -138,9 +143,10 @@ async function runSync(context: InvocationContext, force = false): Promise<{ syn for (const report of reports) { if (!report) continue; - const instanceId = toSafeString(report.instance_id); - const benchmark = toSafeString(report.benchmark ?? instanceId); - const model = toSafeString(report.model); + const fallbackInstanceId = `path:${report.blobPath}`; + const instanceId = toSafeString(report.instance_id) ?? fallbackInstanceId; + const benchmark = toSafeString(report.benchmark) ?? instanceId; + const model = toSafeString(report.model) ?? "unknown"; await tableClient.upsertEntity({ partitionKey: date,