diff --git a/.github/workflows/agentic-ci.yml b/.github/workflows/agentic-ci.yml index 04b192b..8e98da7 100644 --- a/.github/workflows/agentic-ci.yml +++ b/.github/workflows/agentic-ci.yml @@ -111,6 +111,7 @@ jobs: - name: Generate summary run: | + mkdir -p ci-results python3 -c " import json, os, glob results = {} diff --git a/README.md b/README.md index e9d09d9..2ee3757 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ npm install # Start development (backend + frontend) npm run dev +# Verify backend health +curl http://127.0.0.1:3001/api/health + # Open http://localhost:3000 ``` @@ -109,7 +112,9 @@ npm install npm run dev ``` -The backend runs on `http://localhost:3001` and the dashboard on `http://localhost:3000` with hot reload enabled for both. +The backend binds to `127.0.0.1:3001` by default and the dashboard runs on `http://localhost:3000` with hot reload enabled for both. Set `HOST` explicitly only on trusted networks; the local-first server does not provide authentication yet and restricts browser CORS to localhost origins. + +Use `npm run typecheck` to run TypeScript checks for both workspaces without emitting build artifacts. ### Project Structure diff --git a/package-lock.json b/package-lock.json index 8f74cd9..7afcc9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "inference-forge", - "version": "0.1.0", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "inference-forge", - "version": "0.1.0", + "version": "0.5.0", "license": "MIT", "workspaces": [ "packages/server", @@ -5065,7 +5065,7 @@ }, "packages/dashboard": { "name": "@inference-forge/dashboard", - "version": "0.1.0", + "version": "0.5.0", "dependencies": { "react": "^19.0.0", "react-dom": "^19.0.0", @@ -5156,7 +5156,7 @@ }, "packages/server": { "name": "@inference-forge/server", - "version": "0.1.0", + "version": "0.5.0", "dependencies": { "cors": "^2.8.5", "dotenv": "^16.4.0", diff --git a/package.json b/package.json index 65818f9..cec505b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "inference-forge", - "version": "0.1.0", + "version": "0.5.0", "private": true, "description": "All-in-one local LLM inference management suite — monitoring, benchmarking, and Modelfile generation", "workspaces": [ @@ -12,6 +12,7 @@ "dev:server": "npm run dev -w packages/server", "dev:dashboard": "npm run dev -w packages/dashboard", "build": "npm run build -w packages/server && npm run build -w packages/dashboard", + "typecheck": "npm run typecheck -w packages/server && npm run typecheck -w packages/dashboard", "start": "npm run start -w packages/server" }, "devDependencies": { diff --git a/packages/dashboard/package.json b/packages/dashboard/package.json index 59c7b4c..c0feb11 100644 --- a/packages/dashboard/package.json +++ b/packages/dashboard/package.json @@ -1,11 +1,12 @@ { "name": "@inference-forge/dashboard", - "version": "0.1.0", + "version": "0.5.0", "private": true, "type": "module", "scripts": { "dev": "vite --port 3000", "build": "tsc && vite build", + "typecheck": "tsc --noEmit", "preview": "vite preview" }, "dependencies": { diff --git a/packages/dashboard/src/components/Dashboard.tsx b/packages/dashboard/src/components/Dashboard.tsx index 08c3389..0a57e9f 100644 --- a/packages/dashboard/src/components/Dashboard.tsx +++ b/packages/dashboard/src/components/Dashboard.tsx @@ -16,6 +16,7 @@ import { TemplateGallery } from './TemplateGallery'; import { AgentPanel } from './AgentPanel'; import { ModelPull } from './ModelPull'; import { BackendPanel } from './BackendPanel'; +import { APP_VERSION } from '../config/version'; type Tab = 'monitor' | 'benchmark' | 'modelfile' | 'analysis' | 'agents'; @@ -56,7 +57,7 @@ export function Dashboard() { Forge - v0.2.0 + v{APP_VERSION}
diff --git a/packages/dashboard/src/config/version.ts b/packages/dashboard/src/config/version.ts new file mode 100644 index 0000000..8eb8578 --- /dev/null +++ b/packages/dashboard/src/config/version.ts @@ -0,0 +1 @@ +export const APP_VERSION = '0.5.0'; diff --git a/packages/server/package.json b/packages/server/package.json index 5845d30..d0433c1 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,11 +1,12 @@ { "name": "@inference-forge/server", - "version": "0.1.0", + "version": "0.5.0", "private": true, "type": "module", "scripts": { "dev": "tsx watch src/index.ts", "build": "tsc", + "typecheck": "tsc --noEmit", "start": "node dist/index.js" }, "dependencies": { diff --git a/packages/server/src/api/routes.ts b/packages/server/src/api/routes.ts index bf6943e..937a2d6 100644 --- a/packages/server/src/api/routes.ts +++ b/packages/server/src/api/routes.ts @@ -11,6 +11,7 @@ import { hardware } from '../services/hardware.js'; import { throughput } from '../services/throughput.js'; import { alerts } from '../services/alerts.js'; import { perplexity } from '../services/perplexity.js'; +import { benchmarkState } from '../services/benchmark-state.js'; import { promptLibrary } from '../services/prompt-library.js'; import { modelfileLibrary } from '../services/modelfile-library.js'; import { orchestrator, type AgentConfig, type Workflow } from '../services/orchestrator.js'; @@ -18,24 +19,112 @@ import { pressure } from '../services/pressure.js'; import { database } from '../services/database.js'; import { ioProfiler } from '../services/io-profiler.js'; import { costTracker } from '../services/cost-tracker.js'; -import { lmstudio } from '../services/lmstudio.js'; -import { modelRegistry } from '../services/model-registry.js'; +import { lmstudio, type LmsChatMessage } from '../services/lmstudio.js'; +import { modelRegistry, type Backend } from '../services/model-registry.js'; import { storageAdvisor } from '../services/storage-advisor.js'; import { perfProfiler, type ModelProfile } from '../services/perf-profiler.js'; import { routeAdvisor, type RouteRequest } from '../services/route-advisor.js'; import { systemOptimizer } from '../services/system-optimizer.js'; import { openclawBridge } from '../services/openclaw-bridge.js'; import { hivemindBridge } from '../services/hivemind-bridge.js'; +import { APP_VERSION } from '../config/version.js'; export const router = Router(); +type ValidationResult = { ok: true; value: T } | { ok: false; error: string }; + +interface MessageBody { + content: string; +} + +interface OpenClawUsageBody { + agentId: string; + tokens: number; + costUsd: number; +} + +interface LmStudioChatBody { + model: string; + messages: LmsChatMessage[]; + temperature?: number; + max_tokens?: number; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function parseMessageBody(body: unknown): ValidationResult { + if (!isRecord(body) || typeof body.content !== 'string' || body.content.trim().length === 0) { + return { ok: false, error: 'content is required' }; + } + return { ok: true, value: { content: body.content } }; +} + +function parseOpenClawUsageBody(body: unknown): ValidationResult { + if (!isRecord(body) || typeof body.agentId !== 'string' || body.agentId.trim().length === 0) { + return { ok: false, error: 'agentId required' }; + } + const tokens = typeof body.tokens === 'number' && Number.isFinite(body.tokens) ? body.tokens : 0; + const costUsd = typeof body.costUsd === 'number' && Number.isFinite(body.costUsd) ? body.costUsd : 0; + if (tokens < 0 || costUsd < 0) { + return { ok: false, error: 'tokens and costUsd must be non-negative numbers' }; + } + return { ok: true, value: { agentId: body.agentId, tokens, costUsd } }; +} + +function parseLmStudioChatBody(body: unknown): ValidationResult { + if (!isRecord(body) || typeof body.model !== 'string' || body.model.trim().length === 0) { + return { ok: false, error: 'model is required' }; + } + if (!Array.isArray(body.messages) || body.messages.length === 0) { + return { ok: false, error: 'messages must be a non-empty array' }; + } + + const messages: LmsChatMessage[] = []; + for (const message of body.messages) { + if (!isRecord(message)) return { ok: false, error: 'messages must contain objects' }; + if (message.role !== 'system' && message.role !== 'user' && message.role !== 'assistant') { + return { ok: false, error: 'message role must be system, user, or assistant' }; + } + if (typeof message.content !== 'string') { + return { ok: false, error: 'message content must be a string' }; + } + messages.push({ role: message.role, content: message.content }); + } + + const temperature = typeof body.temperature === 'number' && Number.isFinite(body.temperature) + ? body.temperature + : undefined; + const maxTokens = typeof body.max_tokens === 'number' && Number.isFinite(body.max_tokens) + ? body.max_tokens + : undefined; + + if (temperature !== undefined && (temperature < 0 || temperature > 2)) { + return { ok: false, error: 'temperature must be between 0 and 2' }; + } + if (maxTokens !== undefined && (maxTokens < 1 || maxTokens > 32768)) { + return { ok: false, error: 'max_tokens must be between 1 and 32768' }; + } + + return { ok: true, value: { model: body.model, messages, temperature, max_tokens: maxTokens } }; +} + +function isBackend(value: string): value is Backend { + return value === 'ollama' || value === 'lmstudio'; +} + +function writeSse(res: { write: (chunk: string) => void }, payload: unknown): void { + res.write(`data: ${JSON.stringify(payload)}\n\n`); +} + // -- Health --------------------------------------------------------- router.get('/health', async (_req, res) => { const ollamaOnline = await ollama.ping(); res.json({ status: 'ok', - version: '0.5.0', + version: APP_VERSION, ollama: ollamaOnline ? 'connected' : 'disconnected', ollamaUrl: ollama.getBaseUrl(), timestamp: Date.now(), @@ -212,14 +301,14 @@ router.post('/benchmark/run', async (req, res) => { try { const result = await benchmark.run(config); - (globalThis as any).__lastBenchmarkResult = result; + benchmarkState.setBenchmarkResult(result); } catch (err) { console.error('[Benchmark] Error:', err); } }); router.get('/benchmark/result', (_req, res) => { - const result = (globalThis as any).__lastBenchmarkResult; + const result = benchmarkState.getBenchmarkResult(); if (!result) { res.status(404).json({ error: 'No benchmark result available' }); return; @@ -243,15 +332,14 @@ router.post('/benchmark/run-expanded', async (req, res) => { try { const result = await benchmark.runExpanded(config); - (globalThis as any).__lastExpandedBenchmarkResult = result; - (globalThis as any).__lastBenchmarkResult = result; + benchmarkState.setExpandedBenchmarkResult(result); } catch (err) { console.error('[Benchmark] Expanded error:', err); } }); router.get('/benchmark/result-expanded', (_req, res) => { - const result = (globalThis as any).__lastExpandedBenchmarkResult; + const result = benchmarkState.getExpandedBenchmarkResult(); if (!result) { res.status(404).json({ error: 'No expanded benchmark result available' }); return; @@ -260,7 +348,7 @@ router.get('/benchmark/result-expanded', (_req, res) => { }); router.get('/benchmark/export/:format', (req, res) => { - const result = (globalThis as any).__lastBenchmarkResult; + const result = benchmarkState.getBenchmarkResult(); if (!result) { res.status(404).json({ error: 'No benchmark result to export' }); return; @@ -645,23 +733,25 @@ router.post('/sessions/:id/message', async (req, res) => { // Streaming chat endpoint — sends tokens as SSE router.post('/sessions/:id/message/stream', async (req, res) => { - const { content } = req.body as { content: string }; + const parsed = parseMessageBody(req.body); + if (!parsed.ok) { res.status(400).json({ error: parsed.error }); return; } res.setHeader('Content-Type', 'text/event-stream'); res.setHeader('Cache-Control', 'no-cache'); res.setHeader('Connection', 'keep-alive'); - orchestrator.sendMessageStream( + await orchestrator.sendMessageStream( req.params.id, - content, - // TODO: Move to env: (token) - const (token) = process.env.(TOKEN) || ''; + parsed.value.content, + (token) => { + writeSse(res, { type: 'token', content: token }); + }, (fullResponse) => { - res.write(`data: ${JSON.stringify({ type: 'done', content: fullResponse })}\n\n`); + writeSse(res, { type: 'done', content: fullResponse }); res.end(); }, (err) => { - res.write(`data: ${JSON.stringify({ type: 'error', error: String(err) })}\n\n`); + writeSse(res, { type: 'error', error: String(err) }); res.end(); } ); @@ -753,8 +843,12 @@ router.get('/profiles', (_req, res) => { }); router.get('/profiles/:backend/:modelId', (req, res) => { + if (!isBackend(req.params.backend)) { + res.status(400).json({ error: 'backend must be ollama or lmstudio' }); + return; + } const profile = perfProfiler.getProfile( - req.params.backend as any, + req.params.backend, decodeURIComponent(req.params.modelId) ); if (!profile) { res.status(404).json({ error: 'Profile not found' }); return; } @@ -767,8 +861,12 @@ router.post('/profiles/run', async (req, res) => { res.status(400).json({ error: 'backend and modelId are required' }); return; } + if (!isBackend(backend)) { + res.status(400).json({ error: 'backend must be ollama or lmstudio' }); + return; + } try { - const profile = await perfProfiler.profileModel(backend as any, modelId); + const profile = await perfProfiler.profileModel(backend, modelId); res.json({ profile }); } catch (err) { res.status(500).json({ error: String(err) }); @@ -837,10 +935,10 @@ router.get('/openclaw/agents', (_req, res) => { }); router.post('/openclaw/usage', (req, res) => { - // TODO: Move to env: const { agentId, tokens, costUsd } - const const { agentId, tokens, costUsd } = process.env.CONST_{_AGENTID,_TOKENS,_COSTUSD_} || ''; - if (!agentId) { res.status(400).json({ error: 'agentId required' }); return; } - openclawBridge.recordAgentUsage(agentId, tokens || 0, costUsd || 0); + const parsed = parseOpenClawUsageBody(req.body); + if (!parsed.ok) { res.status(400).json({ error: parsed.error }); return; } + const { agentId, tokens, costUsd } = parsed.value; + openclawBridge.recordAgentUsage(agentId, tokens, costUsd); res.json({ success: true }); }); @@ -925,11 +1023,11 @@ router.post('/lmstudio/unload', async (req, res) => { }); router.post('/lmstudio/chat', async (req, res) => { - // TODO: Move to env: const { model, messages, temperature, max_tokens } - const const { model, messages, temperature, max_tokens } = process.env.CONST_{_MODEL,_MESSAGES,_TEMPERATURE,_MAX_TOKENS_} || ''; + const parsed = parseLmStudioChatBody(req.body); + if (!parsed.ok) { res.status(400).json({ error: parsed.error }); return; } + const { model, messages, temperature, max_tokens } = parsed.value; try { - // TODO: Move to env: const result - const const result = process.env.CONST_RESULT || ''; + const result = await lmstudio.chat(model, messages, { temperature, max_tokens }); res.json(result); } catch (err) { res.status(502).json({ error: String(err) }); @@ -938,8 +1036,9 @@ router.post('/lmstudio/chat', async (req, res) => { // Streaming chat for LM Studio router.post('/lmstudio/chat/stream', async (req, res) => { - // TODO: Move to env: const { model, messages, temperature, max_tokens } - const const { model, messages, temperature, max_tokens } = process.env.CONST_{_MODEL,_MESSAGES,_TEMPERATURE,_MAX_TOKENS_} || ''; + const parsed = parseLmStudioChatBody(req.body); + if (!parsed.ok) { res.status(400).json({ error: parsed.error }); return; } + const { model, messages, temperature, max_tokens } = parsed.value; res.setHeader('Content-Type', 'text/event-stream'); res.setHeader('Cache-Control', 'no-cache'); @@ -948,13 +1047,14 @@ router.post('/lmstudio/chat/stream', async (req, res) => { try { await lmstudio.chatStream( model, messages, - // TODO: Move to env: (token) - const (token) = process.env.(TOKEN) || ''; + (token) => { + writeSse(res, { type: 'token', content: token }); + }, { temperature, max_tokens } ); - res.write(`data: ${JSON.stringify({ type: 'done' })}\n\n`); + writeSse(res, { type: 'done' }); } catch (err) { - res.write(`data: ${JSON.stringify({ type: 'error', error: String(err) })}\n\n`); + writeSse(res, { type: 'error', error: String(err) }); } res.end(); }); diff --git a/packages/server/src/config/version.ts b/packages/server/src/config/version.ts new file mode 100644 index 0000000..8eb8578 --- /dev/null +++ b/packages/server/src/config/version.ts @@ -0,0 +1 @@ +export const APP_VERSION = '0.5.0'; diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 04d5ada..f872229 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -4,7 +4,7 @@ import 'dotenv/config'; import express from 'express'; -import cors from 'cors'; +import cors, { type CorsOptions } from 'cors'; import { createServer } from 'http'; import { router } from './api/routes.js'; import { setupWebSocket } from './ws/handler.js'; @@ -14,13 +14,33 @@ import { alerts } from './services/alerts.js'; import { pressure } from './services/pressure.js'; import { database } from './services/database.js'; import { benchmark } from './services/benchmark.js'; +import { benchmarkState } from './services/benchmark-state.js'; import { modelRegistry } from './services/model-registry.js'; +import { APP_VERSION } from './config/version.js'; const PORT = parseInt(process.env.PORT || '3001', 10); +const HOST = process.env.HOST || '127.0.0.1'; const app = express(); +const LOCALHOST_ORIGINS = new Set([ + 'http://localhost:3000', + 'http://127.0.0.1:3000', + `http://localhost:${PORT}`, + `http://127.0.0.1:${PORT}`, +]); + +const corsOptions: CorsOptions = { + origin(origin, callback) { + if (!origin || LOCALHOST_ORIGINS.has(origin)) { + callback(null, true); + return; + } + callback(new Error(`CORS origin not allowed: ${origin}`)); + }, +}; + // Middleware -app.use(cors()); +app.use(cors(corsOptions)); app.use(express.json({ limit: '10mb' })); // API routes @@ -68,17 +88,20 @@ alerts.subscribe((alert) => { // Persist benchmark results (via progress subscription to detect completion) benchmark.subscribeProgress((message, progress) => { if (progress >= 1) { - const result = (globalThis as any).__lastBenchmarkResult; + const result = benchmarkState.getBenchmarkResult(); if (result) database.saveBenchmarkRun(result); } }); -server.listen(PORT, () => { +server.listen(PORT, HOST, () => { + if (HOST !== '127.0.0.1' && HOST !== 'localhost') { + console.warn(`[Security] Inference Forge is bound to ${HOST}. This local-first server has no authentication; expose only on trusted networks.`); + } console.log(` +---------------------------------------+ - | Inference Forge v0.3.0 | - | http://localhost:${PORT} | - | WebSocket: ws://localhost:${PORT}/ws | + | Inference Forge v${APP_VERSION} | + | http://${HOST}:${PORT} | + | WebSocket: ws://${HOST}:${PORT}/ws | +---------------------------------------+ `); }); diff --git a/packages/server/src/services/benchmark-state.ts b/packages/server/src/services/benchmark-state.ts new file mode 100644 index 0000000..25ab37f --- /dev/null +++ b/packages/server/src/services/benchmark-state.ts @@ -0,0 +1,25 @@ +import type { BenchmarkSummary, ExpandedBenchmarkSummary } from './benchmark.js'; + +class BenchmarkStateService { + private lastBenchmarkResult: BenchmarkSummary | ExpandedBenchmarkSummary | null = null; + private lastExpandedBenchmarkResult: ExpandedBenchmarkSummary | null = null; + + setBenchmarkResult(result: BenchmarkSummary | ExpandedBenchmarkSummary): void { + this.lastBenchmarkResult = result; + } + + getBenchmarkResult(): BenchmarkSummary | ExpandedBenchmarkSummary | null { + return this.lastBenchmarkResult; + } + + setExpandedBenchmarkResult(result: ExpandedBenchmarkSummary): void { + this.lastExpandedBenchmarkResult = result; + this.setBenchmarkResult(result); + } + + getExpandedBenchmarkResult(): ExpandedBenchmarkSummary | null { + return this.lastExpandedBenchmarkResult; + } +} + +export const benchmarkState = new BenchmarkStateService(); diff --git a/packages/server/src/services/benchmark.ts b/packages/server/src/services/benchmark.ts index 4ab211c..92861b8 100644 --- a/packages/server/src/services/benchmark.ts +++ b/packages/server/src/services/benchmark.ts @@ -92,6 +92,7 @@ export interface ExpandedBenchmarkSummary { } type ProgressCallback = (message: string, progress: number) => void; +type ExpandedParameterKey = 'numGpu' | 'numThread' | 'numCtx' | 'numBatch'; // -- Standard benchmark prompts ------------------------------------- @@ -178,8 +179,7 @@ export class BenchmarkService { const summary = config.kvCacheTypes.map((kvType) => { const kvResults = results.filter((r) => r.kvCacheType === kvType); - // TODO: Move to env: const avgTps - const const avgTps = process.env.CONST_AVGTPS || ''; + const avgTps = average(kvResults.map((r) => r.tokensPerSecond)); const avgTotal = kvResults.reduce((s, r) => s + r.totalDurationMs, 0) / kvResults.length; const avgEval = kvResults.reduce((s, r) => s + r.evalDurationMs, 0) / kvResults.length; @@ -328,7 +328,7 @@ export class BenchmarkService { }; const mode = modeMap[paramName] || 'kv-cache'; - const metaKey: Record = { + const metaKey: Record = { num_gpu: 'numGpu', num_thread: 'numThread', num_ctx: 'numCtx', @@ -350,8 +350,9 @@ export class BenchmarkService { mode, run: run + 1, }; - if (metaKey[paramName]) { - (expanded as any)[metaKey[paramName]] = value; + const key = metaKey[paramName]; + if (key) { + setExpandedParameter(expanded, key, value); } results.push(expanded); } @@ -360,7 +361,7 @@ export class BenchmarkService { const summaryItems = steps.map((value) => { const key = metaKey[paramName]; - const filtered = results.filter((r) => (r as any)[key] === value); + const filtered = results.filter((r) => key ? r[key] === value : false); return this.buildSummaryItem(`${paramName}=${value}`, paramName, value, filtered); }); @@ -374,8 +375,7 @@ export class BenchmarkService { results: ExpandedBenchmarkResult[] ): ExpandedBenchmarkSummary['summary'][0] { const n = results.length || 1; - // TODO: Move to env: const avgTps - const const avgTps = process.env.CONST_AVGTPS || ''; + const avgTps = average(results.map((r) => r.tokensPerSecond)); const avgTotal = results.reduce((s, r) => s + r.totalDurationMs, 0) / n; const avgEval = results.reduce((s, r) => s + r.evalDurationMs, 0) / n; const avgPromptEval = results.reduce((s, r) => { @@ -412,8 +412,7 @@ export class BenchmarkService { const promptEvalDurationMs = (response.prompt_eval_duration || 0) / 1_000_000; const evalDurationMs = (response.eval_duration || 0) / 1_000_000; const evalCount = response.eval_count || 0; - // TODO: Move to env: const tokensPerSecond - const const tokensPerSecond = process.env.CONST_TOKENSPERSECOND || ''; + const tokensPerSecond = evalDurationMs > 0 ? evalCount / (evalDurationMs / 1000) : 0; return { model, @@ -432,4 +431,17 @@ export class BenchmarkService { } } +function average(values: number[]): number { + if (values.length === 0) return 0; + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function setExpandedParameter( + result: ExpandedBenchmarkResult, + key: ExpandedParameterKey, + value: number +): void { + result[key] = value; +} + export const benchmark = new BenchmarkService(); diff --git a/packages/server/src/services/route-advisor.ts b/packages/server/src/services/route-advisor.ts index a23d823..3a51fb2 100644 --- a/packages/server/src/services/route-advisor.ts +++ b/packages/server/src/services/route-advisor.ts @@ -6,7 +6,7 @@ import { perfProfiler, type ModelProfile } from './perf-profiler.js'; import { modelRegistry, type Backend } from './model-registry.js'; -import { pressure } from './pressure.js'; +import { pressure, type ResourcePressure } from './pressure.js'; // -- Types ---------------------------------------------------------- @@ -138,13 +138,10 @@ export class RouteAdvisor { } } - // Estimate performance const tokS = profile.tokSGpu || profile.tokSCpu || 1; - // TODO: Move to env: const firstToken - const const firstToken = process.env.CONST_FIRSTTOKEN || ''; + const firstToken = estimateFirstTokenMs(profile); - // TODO: Move to env: if (firstToken > latencyLimit && request.latency ! - const if (firstToken > latencyLimit && request.latency ! = process.env.IF_(FIRSTTOKEN_>_LATENCYLIMIT_&&_REQUEST.LATENCY_! || ''; + if (firstToken > latencyLimit && request.latency !== 'batch') { warnings.push(`First token (~${Math.round(firstToken)}ms) may exceed ${request.latency} budget`); } @@ -168,7 +165,7 @@ export class RouteAdvisor { request: RouteRequest, affinityArchs: string[], latencyLimit: number, - pressureState: any + pressureState: ResourcePressure | null ): number { let score = 50; // baseline @@ -207,9 +204,7 @@ export class RouteAdvisor { score += request.latency === 'realtime' ? 20 : request.latency === 'interactive' ? 10 : 2; } - // Latency penalty - // TODO: Move to env: const firstToken - const const firstToken = process.env.CONST_FIRSTTOKEN || ''; + const firstToken = estimateFirstTokenMs(profile); if (firstToken > latencyLimit) { score -= 20; } @@ -277,4 +272,10 @@ export class RouteAdvisor { } } +function estimateFirstTokenMs(profile: ModelProfile): number { + if (profile.firstTokenMs !== null) return profile.firstTokenMs; + const tokS = profile.tokSGpu || profile.tokSCpu || 0; + return tokS > 0 ? Math.round(1000 / tokS) : Infinity; +} + export const routeAdvisor = new RouteAdvisor();