Skip to content

feat(bench): Add opt-out toggle for community benchmark upload #1052

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jabagawee wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(bench): Add opt-out toggle for community benchmark upload #1052

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 20 additions & 15 deletions omlx/admin/benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ class BenchmarkRequest(BaseModel):
prompt_lengths: list[int]
generation_length: int = 128
batch_sizes: list[int] = []
should_upload_results: bool = True
@field_validator("prompt_lengths")
@classmethod
def validate_prompt_lengths(cls, v: list[int]) -> list[int]:
Expand Down Expand Up @@ -732,21 +733,25 @@ async def run_benchmark(run: BenchmarkRun, engine_pool: Any) -> None:
},
})

# Upload results to omlx.ai (failures don't affect benchmark status)
try:
await _upload_to_omlx_ai(run, engine_pool)
except Exception as e:
logger.warning(f"Benchmark upload to omlx.ai failed: {e}")
await _send_event(run, {
"type": "upload_done",
"data": {
"owner_hash": None,
"total": 0,
"success": 0,
"failed": 0,
"error": str(e),
},
})
# Upload results to omlx.ai (failures don't affect benchmark status).
# Skipped when the user opted out via the dashboard checkbox; the
# frontend's 'done' handler finalizes directly in that case, so no
# additional SSE event is needed here.
if run.request.should_upload_results:
try:
await _upload_to_omlx_ai(run, engine_pool)
except Exception as e:
logger.warning(f"Benchmark upload to omlx.ai failed: {e}")
await _send_event(run, {
"type": "upload_done",
"data": {
"owner_hash": None,
"total": 0,
"success": 0,
"failed": 0,
"error": str(e),
},
})

except asyncio.CancelledError:
run.status = "cancelled"
Expand Down
3 changes: 3 additions & 0 deletions omlx/admin/i18n/en.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,9 @@
"bench.config.generation_hint": "Generation length: 128 tokens (fixed)",
"bench.config.batch_tests": "Continuous Batching Tests",
"bench.config.batch_hint": "Batch tests use pp1024 / tg128",
"bench.config.share_results": "Share results with the omlx.ai community",
"bench.config.share_results_hint": "Uploads chip, RAM, GPU, OS version, model name, quantization, and performance numbers to omlx.ai/api/benchmarks. Includes a stable per-device hash so you can find your submissions at omlx.ai/my/<hash>. Uncheck to skip the upload for this run (e.g. when benchmarking a private model).",
"bench.upload.in_progress": "Uploading to community benchmarks...",
"bench.config.run_button": "Run Benchmark",
"bench.progress.preparing": "Preparing...",
"bench.progress.cancel": "Cancel",
Expand Down
27 changes: 19 additions & 8 deletions omlx/admin/static/js/dashboard.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,7 @@
benchModelId: '',
benchPromptLengths: { 1024: true, 4096: true, 8192: false, 16384: false, 32768: false, 65536: false, 131072: false, 200000: false },
benchBatchSizes: { 2: true, 4: true, 8: false },
benchShouldUploadResults: true,
benchRunning: false,
benchBenchId: null,
benchProgress: null,
Expand Down Expand Up @@ -2098,6 +2099,7 @@
prompt_lengths: promptLengths,
generation_length: 128,
batch_sizes: batchSizes,
should_upload_results: this.benchShouldUploadResults,
}),
});

Expand Down Expand Up @@ -2149,15 +2151,24 @@
this.benchBatchResults = [...this.benchBatchResults, data.data];
}
} else if (data.type === 'done') {
// Benchmark tests done, uploading starts
this.benchUploading = true;
this.benchProgress = {
phase: 'upload',
message: 'Uploading to community benchmarks...',
current: 0,
total: 0,
};
// Benchmark tests done. If the user opted in to
// sharing results, the server follows up with
// upload events; otherwise we finalize here.
this.loadModels();
if (this.benchShouldUploadResults) {
this.benchUploading = true;
this.benchProgress = {
phase: 'upload',
message: window.t('bench.upload.in_progress'),
current: 0,
total: 0,
};
} else {
this.benchRunning = false;
this.benchProgress = null;
es.close();
this.benchEventSource = null;
}
} else if (data.type === 'upload') {
this.benchUploadResults = [...this.benchUploadResults, data.data];
} else if (data.type === 'upload_done') {
Expand Down
12 changes: 12 additions & 0 deletions omlx/admin/templates/dashboard/_bench.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,18 @@ <h3 class="text-2xl font-bold tracking-tight text-neutral-900">{{ t('bench.headi
<p class="text-xs text-neutral-400 mt-1.5">{{ t('bench.config.batch_hint') }}</p>
</div>

<!-- Share results consent -->
<div>
<label class="flex items-start gap-2 cursor-pointer">
<input type="checkbox"
x-model="benchShouldUploadResults"
:disabled="benchRunning"
class="w-4 h-4 mt-0.5 rounded border-neutral-300 text-neutral-900 focus:ring-neutral-900">
<span class="text-sm text-neutral-700">{{ t('bench.config.share_results') }}</span>
</label>
<p class="text-xs text-neutral-400 mt-1.5 ml-6">{{ t('bench.config.share_results_hint') }}</p>
</div>

<!-- Run / Cancel Button -->
<div class="flex items-center gap-3 pt-2">
<button x-show="!benchRunning"
Expand Down