diff --git a/README.md b/README.md index 8cb8ed323d..773279861c 100644 --- a/README.md +++ b/README.md @@ -764,6 +764,9 @@ is active. Throughput defaults live in ClawSweeper has one main capacity knob: `config/automation-limits.json` -> `workers.max`. The current value is `128`. +This is a Codex worker budget, not a GitHub Actions runner limit. Deterministic +exact-review publishers, comment routers, and lease reconcilers are shown as +control-plane workflows and do not consume these 128 slots. Lane limits are derived from that number: normal review defaults to 89 shards for manual/backstop and scheduled runs, hot intake up to 44 shards, commit review 6 commits per page, and existing repair/issue implementation lanes use diff --git a/dashboard/worker.ts b/dashboard/worker.ts index f4c80b8309..8caacc8349 100644 --- a/dashboard/worker.ts +++ b/dashboard/worker.ts @@ -3157,6 +3157,18 @@ function exactReviewQueueStats( publicationCapacity, publicationDispatchLeaseMs, ); + const lanes = { + review: exactReviewQueueLaneStats( + items.filter((item) => !exactReviewQueueIsPublication(item)), + now, + capacity, + ), + publication: exactReviewQueueLaneStats( + items.filter(exactReviewQueueIsPublication), + now, + publicationCapacity, + ), + }; return { pending: handoffHealth.phases.pending.count, dispatching: handoffHealth.phases.dispatching.count, @@ -3168,6 +3180,7 @@ function exactReviewQueueStats( oldest_leased_at: handoffHealth.phases.leased.oldest_at, oldest_leased_age_seconds: handoffHealth.phases.leased.oldest_age_seconds, handoff_health: handoffHealth, + lanes, next_wake_at: nextWakeAt === null ? null : new Date(nextWakeAt).toISOString(), dispatcher: { state: state.dispatcher?.state || "unknown", @@ -3182,6 +3195,35 @@ function exactReviewQueueStats( }; } +function exactReviewQueueLaneStats(items: ExactReviewQueueItem[], now: number, capacity: number) { + const pendingItems = items.filter((item) => item.state === "pending"); + const dispatchingItems = items.filter((item) => item.state === "dispatching"); + const leasedItems = items.filter((item) => item.state === "leased"); + const active = dispatchingItems.length + leasedItems.length; + const oldestPendingAt = pendingItems.reduce( + (oldest, item) => (oldest === null ? item.createdAt : Math.min(oldest, item.createdAt)), + null, + ); + const nextAttemptAt = pendingItems.reduce( + (next, item) => (next === null ? item.nextAttemptAt : Math.min(next, item.nextAttemptAt)), + null, + ); + return { + pending: pendingItems.length, + ready: pendingItems.filter((item) => item.nextAttemptAt <= now).length, + backoff: pendingItems.filter((item) => item.nextAttemptAt > now).length, + dispatching: dispatchingItems.length, + leased: leasedItems.length, + capacity, + active, + available_slots: Math.max(0, capacity - active), + oldest_pending_at: oldestPendingAt === null ? null : new Date(oldestPendingAt).toISOString(), + oldest_pending_age_seconds: + oldestPendingAt === null ? null : Math.max(0, Math.floor((now - oldestPendingAt) / 1_000)), + next_attempt_at: nextAttemptAt === null ? null : new Date(nextAttemptAt).toISOString(), + }; +} + export function exactReviewQueueNextWakeAt( state: ExactReviewQueueState, now: number, @@ -3894,6 +3936,7 @@ async function statusSnapshot(env) { ]).sort(newestWorkflowRunFirst); const workerRuns = activeRuns.filter((run) => !isSupportWorkflowRun(run)); const supportRuns = activeRuns.filter((run) => isSupportWorkflowRun(run)); + const controlPlane = controlPlaneSnapshot(activeRuns); const operationalHealth = summarizeOperationalHealth( activeRunCandidates.filter((run) => !isSupportWorkflowRun(run)), generatedAt, @@ -3903,7 +3946,7 @@ async function statusSnapshot(env) { (run) => run.status === "completed" && !isSupportWorkflowRun(run) && - codexJobName(`${run.name || ""} ${run.display_title || ""}`) && + isCodexWorkflowFallback(run) && TERMINAL_BAD_CONCLUSIONS.has(String(run.conclusion)), ); const activeJobs = await activeWorkerSnapshot(env, repo, workerRuns, github); @@ -4004,6 +4047,7 @@ async function statusSnapshot(env) { worker_detail_runs: activeJobs.detailRuns, worker_detail_fallbacks: activeJobs.fallbacks, }, + control_plane: controlPlane, health: publicWorkerHealth, operational_health: operationalHealth, averages: { @@ -4960,7 +5004,7 @@ async function activeWorkerSnapshot( for (const result of results) { if (result.error) { errors.push(`workflow jobs ${result.run.id}: ${result.error}`); - if (codexJobName(`${result.run.name || ""} ${result.run.display_title || ""}`)) { + if (isCodexWorkflowFallback(result.run)) { workers.push(normalizeFallbackWorker(result.run)); fallbacks += 1; } @@ -4968,16 +5012,13 @@ async function activeWorkerSnapshot( } if (result.workers.length) { workers.push(...result.workers); - } else if ( - !result.hasWorkerJobs && - codexJobName(`${result.run.name || ""} ${result.run.display_title || ""}`) - ) { + } else if (!result.hasWorkerJobs && isCodexWorkflowFallback(result.run)) { workers.push(normalizeFallbackWorker(result.run)); fallbacks += 1; } } for (const run of runs.slice(detailRunLimit)) { - if (!codexJobName(`${run.name || ""} ${run.display_title || ""}`)) continue; + if (!isCodexWorkflowFallback(run)) continue; workers.push(normalizeFallbackWorker(run)); fallbacks += 1; } @@ -5019,9 +5060,7 @@ async function recentWorkerHealth( const completedRuns = runs .filter( (run) => - run.status === "completed" && - !isSupportWorkflowRun(run) && - codexJobName(`${run.name || ""} ${run.display_title || ""}`), + run.status === "completed" && !isSupportWorkflowRun(run) && isCodexWorkflowFallback(run), ) .sort(newestWorkflowRunFirst) .slice(0, RECENT_WORKER_HEALTH_RUN_LIMIT); @@ -7662,8 +7701,40 @@ function workflowRunSummary(run) { }; } -function codexJobName(name) { - return /review|codex|repair|worker|commit/i.test(name); +function isCodexWorkflowFallback(run) { + const name = `${run?.name || ""} ${run?.display_title || ""}`; + if ( + /repair comment router|clawsweeper_comment|@publish:|publish exact review artifact|exact.review publication|reconcile exact.review lease|sync codex review comments/i.test( + name, + ) + ) { + return false; + } + return /review clawsweeper items|review hot (?:clawsweeper items|target repo)|review target repo|review event items?|retry failed codex reviews|commit review|repair cluster|automerge repair|issue implementation|assist\b/i.test( + name, + ); +} + +function controlPlaneSnapshot(runs) { + const snapshot = { + publishers: { running: 0, waiting: 0 }, + comment_routers: { running: 0, waiting: 0 }, + reconcilers: { running: 0, waiting: 0 }, + }; + for (const run of runs) { + const name = `${run?.name || ""} ${run?.display_title || ""}`; + const lane = /@publish:|publish exact review artifact|exact.review publication/i.test(name) + ? snapshot.publishers + : /repair comment router|clawsweeper_comment|sync codex review comments/i.test(name) + ? snapshot.comment_routers + : /reconcile exact.review lease/i.test(name) + ? snapshot.reconcilers + : null; + if (!lane) continue; + if (run.status === "in_progress") lane.running += 1; + else lane.waiting += 1; + } + return snapshot; } function laneRank(mode) { @@ -9088,6 +9159,14 @@ h2::before { content: ""; flex: 0 0 auto; width: 14px; height: 2px; border-radiu line-height: 1.4; } .capacity-rail { margin-top: 30px; } +.overview-section-title { + margin: 28px 0 0; + color: var(--muted); + font-size: 10px; + font-weight: 700; + letter-spacing: 0.11em; + text-transform: uppercase; +} .capacity-bar { display: flex; height: 10px; @@ -9099,6 +9178,33 @@ h2::before { content: ""; flex: 0 0 auto; width: 14px; height: 2px; border-radiu .capacity-bar .active { background: var(--claw); } .capacity-bar .waiting { background: var(--amber); } .capacity-meta { margin-top: 8px; color: var(--muted); font-size: 12px; } +.capacity-note { margin-top: 5px; color: var(--muted); font-size: 11px; } +.exact-lanes { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px; + margin-top: 10px; +} +.exact-lane, +.control-plane { + padding: 14px; + border: 1px solid var(--line); + border-radius: 10px; + background: var(--panel); +} +.exact-lane-head { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; } +.exact-lane-head strong { font-size: 13px; } +.exact-lane-head span { color: var(--muted); font-size: 11px; } +.lane-counts { display: grid; gap: 6px; margin-top: 12px; } +.lane-count { display: flex; justify-content: space-between; gap: 12px; color: var(--muted); font-size: 11px; } +.lane-count strong { color: var(--text); font-weight: 600; } +.lane-bar { height: 6px; margin-top: 12px; overflow: hidden; border-radius: 999px; background: var(--track); } +.lane-bar i { display: block; height: 100%; background: var(--claw); } +.lane-foot { margin-top: 7px; color: var(--muted); font-size: 11px; } +.control-plane { display: grid; gap: 8px; margin-top: 10px; } +.control-plane-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 12px; font-size: 12px; } +.control-plane-row span:last-child { color: var(--muted); } +.control-plane-note { margin-top: 3px; color: var(--muted); font-size: 11px; line-height: 1.45; } .exact-handoff { margin-top: 18px; padding: 14px; @@ -9714,6 +9820,7 @@ a.pill:hover { color: var(--claw); text-decoration: none; } .worker-progress { display: none; } .exact-handoff-head, .handoff-foot { align-items: start; flex-direction: column; } .handoff-phases { grid-template-columns: 1fr; } + .exact-lanes { grid-template-columns: 1fr; } dialog { margin: 7px; max-height: calc(100vh - 14px); } } @@ -9753,7 +9860,13 @@ a.pill:hover { color: var(--claw); text-decoration: none; } Live control-plane telemetry
+

Codex Capacity

+

Exact Review

+
+

Control Plane · GitHub Actions, not Codex

+
+

Handoff Health

@@ -10167,25 +10280,65 @@ function renderSystemMap(data) { document.getElementById("flow-map").innerHTML = nodes.map(node => '
' + esc(node[0]) + '' + fmt.format(node[1]) + '

' + esc(node[2]) + '

' ).join(""); + const budget = Math.max(0, fleet.worker_budget || 0); const running = workers.filter(worker => worker.status === "in_progress").length; const waiting = workers.length - running; - const budget = Math.max(0, fleet.worker_budget || 0); const free = Math.max(0, budget - running - waiting); + const overflow = Math.max(0, running + waiting - budget); const share = value => budget ? Math.min(100, (value / budget) * 100) : 0; document.getElementById("capacity-rail").innerHTML = '
' + - '
' + fmt.format(running) + ' running · ' + fmt.format(waiting) + ' waiting · ' + fmt.format(free) + ' of ' + fmt.format(budget) + ' slots free
'; + '
' + fmt.format(running) + ' running · ' + fmt.format(waiting) + ' waiting · ' + fmt.format(free) + ' of ' + fmt.format(budget) + ' Codex slots free' + (overflow ? ' · ' + fmt.format(overflow) + ' over budget' : '') + '
' + + '
Only jobs that execute Codex count against this budget.
'; const fallbacks = fleet.worker_detail_fallbacks || 0; document.getElementById("overview-note").textContent = fallbacks ? "Live jobs with " + fallbacks + " workflow fallback" + (fallbacks === 1 ? "" : "s") : "Live GitHub job and step telemetry"; } +function renderExactReviewLanes(queue) { + const target = document.getElementById("exact-review-lanes"); + if (!target) return; + const lanes = queue?.lanes; + if (!lanes?.review || !lanes?.publication) { + target.innerHTML = '
Exact-review lane telemetry unavailable.
'; + return; + } + target.innerHTML = [["Review admission", lanes.review], ["Result publication", lanes.publication]].map(([label, lane]) => { + const capacity = Math.max(0, lane.capacity || 0); + const active = Math.max(0, lane.active || 0); + const used = capacity ? Math.min(100, (active / capacity) * 100) : 0; + const oldest = Number.isFinite(lane.oldest_pending_age_seconds) + ? " · oldest " + elapsed(lane.oldest_pending_age_seconds * 1000) + : ""; + return '
' + esc(label) + '' + fmt.format(active) + ' of ' + fmt.format(capacity) + ' active
' + + '
' + + '
Pending' + fmt.format(lane.pending || 0) + '
' + + '
Ready' + fmt.format(lane.ready || 0) + '
' + + '
Backoff' + fmt.format(lane.backoff || 0) + '
' + + '
Dispatching' + fmt.format(lane.dispatching || 0) + '
' + + '
Leased' + fmt.format(lane.leased || 0) + '
' + + '
' + + '
' + fmt.format(lane.available_slots || 0) + ' ' + esc(label.toLowerCase()) + ' slots open' + esc(oldest) + '
'; + }).join(""); +} +function renderControlPlane(controlPlane, workerBudget) { + const target = document.getElementById("control-plane"); + if (!target) return; + const rows = [ + ["Exact publishers", controlPlane?.publishers], + ["Comment routers", controlPlane?.comment_routers], + ["Lease reconcilers", controlPlane?.reconcilers] + ]; + target.innerHTML = rows.map(([label, lane]) => + '
' + esc(label) + '' + fmt.format(lane?.running || 0) + ' running · ' + fmt.format(lane?.waiting || 0) + ' waiting
' + ).join("") + '
These workflows consume GitHub runners but not the ' + fmt.format(workerBudget || 0) + '-slot Codex budget.
'; +} function renderExactReviewHandoff(queue) { const target = document.getElementById("exact-review-handoff"); if (!target) return; const health = queue?.handoff_health; if (!health?.phases) { - target.innerHTML = '
Exact-review handoffQueue telemetry unavailable in this snapshot.
unknown
'; + target.innerHTML = '
Queue handoff healthQueue telemetry unavailable in this snapshot.
unknown
'; return; } const status = ["idle", "healthy", "degraded", "stalled"].includes(health.status) ? health.status : "unknown"; @@ -10203,7 +10356,7 @@ function renderExactReviewHandoff(queue) { }).join(""); const slots = fmt.format(health.available_slots || 0) + " of " + fmt.format(health.capacity || 0) + " exact-review slots open"; const threshold = "stalled after " + elapsed((health.stalled_after_seconds || 0) * 1000); - target.innerHTML = '
Exact-review handoff' + esc(health.message || "Queue phase telemetry") + '
' + esc(status) + '
' + phases + '
' + esc(slots) + '' + esc(threshold) + '
'; + target.innerHTML = '
Queue handoff health' + esc(health.message || "Queue phase telemetry") + '
' + esc(status) + '
' + phases + '
' + esc(slots) + '' + esc(threshold) + '
'; } function renderWorkers(rows) { workerIndex = new Map(rows.map(worker => [String(worker.id), worker])); @@ -10417,15 +10570,17 @@ function renderDashboard(data, note) { const fleet = data.fleet; const operational = data.operational_health || {}; document.getElementById("metrics").innerHTML = [ - metric("Claw Workers", fmt.format(fleet.active_codex_jobs), "budget " + fleet.worker_budget, fleet.budget_used_percent, "var(--green)"), + metric("Codex Workers", fmt.format(fleet.active_codex_jobs), "Codex budget " + fleet.worker_budget, fleet.budget_used_percent, "var(--green)"), metric("Active Sweeps", fmt.format(fleet.active_workflow_runs), fmt.format(operational.running_over_threshold || 0) + " over 150m", Math.min(100, fleet.active_workflow_runs * 3), operational.running_over_threshold ? "var(--red)" : "var(--claw)"), metric("Queue Depth", fmt.format(fleet.queued_workflow_runs), fmt.format(operational.queued_over_threshold || 0) + " over 30m", Math.min(100, fleet.queued_workflow_runs * 10), operational.queued_over_threshold ? "var(--amber)" : "var(--green)"), metric("Error Rate", (data.health?.error_rate_percent || 0) + "%", fmt.format(data.health?.failed_attempts || 0) + " failed / " + fmt.format(data.health?.attempts || 0) + " attempts", Math.min(100, data.health?.error_rate_percent || 0), data.health?.failed_attempts ? "var(--red)" : "var(--green)"), metric("Recovery Rate", data.health?.recovery_rate_percent == null ? "n/a" : data.health.recovery_rate_percent + "%", fmt.format(data.health?.unresolved_failures || 0) + " unresolved", data.health?.recovery_rate_percent == null ? 100 : data.health.recovery_rate_percent, data.health?.unresolved_failures ? "var(--amber)" : "var(--green)"), - metric("Capacity", fleet.budget_used_percent + "%", "fleet utilization", fleet.budget_used_percent, "var(--green)") + metric("Codex Capacity", fleet.budget_used_percent + "%", "Codex slot utilization", fleet.budget_used_percent, "var(--green)") ].join(""); renderHealthHistory(data.operational_health); renderSystemMap(data); + renderExactReviewLanes(data.exact_review_queue); + renderControlPlane(data.control_plane, fleet.worker_budget); renderExactReviewHandoff(data.exact_review_queue); renderApplyHealth(data); renderAutomaticWork(data.automatic_work || []); diff --git a/docs/limits.md b/docs/limits.md index 92cd2f4d89..c01345c15c 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -22,6 +22,8 @@ docs stay in sync with the derived budget. The mental model: - `workers.max` is the global Codex capacity budget. +- GitHub Actions workflows that only route comments, publish exact-review results, + or reconcile leases do not execute Codex and do not consume that budget. - Priority lanes are repair, issue implementation, and exact-item review. - Background lanes are normal review, hot intake, and commit review. - Assist has a small fixed cap because it is lightweight maintainer Q&A, not a @@ -124,6 +126,14 @@ backlog drain. Exact capacity is consumed only while queue work is pending. As those priority workers start, normal, hot-intake, and commit-review planners count them and reduce their next background wave. +Exact-review result publication has a separate 24-workflow Actions lane. Its +checkout, artifact handling, comment sync, and result routing are deterministic +control-plane work: they consume GitHub runners, but not Codex slots. The +comment router and the singleton lease reconciler follow the same accounting +rule. Dashboard Codex capacity therefore counts only jobs whose steps execute +Codex; it reports these control-plane workflows separately instead of deducting +them from `workers.max`. + Each dispatched workflow claims its opaque lease before checkout. Protocol v2 binds claim and completion to the item key, lease revision, run attempt, claim generation, and an immutable decision snapshot. During the rolling-upgrade diff --git a/docs/live-dashboard.md b/docs/live-dashboard.md index c6aa0ee199..d1be0335ca 100644 --- a/docs/live-dashboard.md +++ b/docs/live-dashboard.md @@ -284,6 +284,15 @@ so handoff recovery stays live. If the optional queue read fails, it reports `exact_review_queue: null` and `diagnostics.exact_review_queue_error` without making the otherwise-current fleet snapshot eligible for stale fallback. +For capacity displays, `/api/exact-review-queue` also exposes compatible +`lanes.review` and `lanes.publication` objects. Each lane reports its own +pending, ready, backoff, dispatching, leased, capacity, active, available-slot, +oldest-pending, and next-attempt values. The existing top-level aggregate fields +remain available for older consumers. `/api/status` separately reports active +exact publishers, comment routers, and lease reconcilers under `control_plane`; +those are GitHub Actions workflows, not Codex workers, and never reduce the +128-slot Codex capacity rail. + Executors report the GitHub job outcome from their finalizer. Failure or cancellation clears the lease and requeues the item. Finalizer success remains provisional because GitHub can still cancel the run or fail a post-action; only diff --git a/test/dashboard-worker.test.ts b/test/dashboard-worker.test.ts index 7186a5a14d..ac163c087f 100644 --- a/test/dashboard-worker.test.ts +++ b/test/dashboard-worker.test.ts @@ -81,19 +81,74 @@ test("exact-review queue admits and wakes up to 24 publishers", () => { }); test("dashboard status reads the exact-review handoff model from the durable queue", async () => { - const queue = new ExactReviewQueue({ storage: new MemoryDurableStorage() }, {}); + const storage = new MemoryDurableStorage(); + const queue = new ExactReviewQueue({ storage }, {}); await queue.fetch(buildExactReviewQueueRequest("handoff-status", 597, "opened")); + await queue.fetch(buildExactReviewQueueRequest("backoff-status", 598, "opened")); + await queue.fetch(buildExactReviewQueueRequest("leased-review-status", 600, "opened")); + await queue.fetch( + buildExactReviewQueueRequest( + "publication-status", + 599, + "exact_review_artifact_publish", + "issue", + undefined, + exactReviewPublicationOverrides(599, "5990"), + ), + ); + const state = (await storage.get("exact-review-queue")) as { + items: Record< + string, + { + state: "pending" | "dispatching" | "leased"; + nextAttemptAt: number; + leaseId?: string; + leaseExpiresAt?: number; + } + >; + }; + state.items["openclaw/gogcli#598"].nextAttemptAt = Date.now() + 60_000; + for (const key of ["openclaw/gogcli#600", "openclaw/gogcli#599@publish:5990:1"]) { + state.items[key].state = "leased"; + state.items[key].leaseId = `lease-${key}`; + state.items[key].leaseExpiresAt = Date.now() + 60_000; + } + await storage.put("exact-review-queue", state); const status = await exactReviewQueueStatusSnapshot({ EXACT_REVIEW_QUEUE: new MemoryDurableNamespace(queue), }); assert.ok(status); - assert.equal(status.pending, 1); + assert.equal(status.pending, 2); assert.equal(status.dispatching, 0); - assert.equal(status.leased, 0); + assert.equal(status.leased, 2); assert.equal(status.handoff_health.status, "healthy"); - assert.equal(status.handoff_health.phases.pending.count, 1); + assert.equal(status.handoff_health.phases.pending.count, 2); + assert.deepEqual( + { + pending: status.lanes.review.pending, + ready: status.lanes.review.ready, + backoff: status.lanes.review.backoff, + active: status.lanes.review.active, + available_slots: status.lanes.review.available_slots, + capacity: status.lanes.review.capacity, + }, + { pending: 2, ready: 1, backoff: 1, active: 1, available_slots: 63, capacity: 64 }, + ); + assert.deepEqual( + { + pending: status.lanes.publication.pending, + ready: status.lanes.publication.ready, + backoff: status.lanes.publication.backoff, + active: status.lanes.publication.active, + available_slots: status.lanes.publication.available_slots, + capacity: status.lanes.publication.capacity, + }, + { pending: 0, ready: 0, backoff: 0, active: 1, available_slots: 23, capacity: 24 }, + ); + assert.equal(typeof status.lanes.review.oldest_pending_at, "string"); + assert.equal(typeof status.lanes.review.next_attempt_at, "string"); assert.equal(await exactReviewQueueStatusSnapshot({}), null); }); @@ -4667,7 +4722,7 @@ test("dashboard HTML preserves UTF-8 emoji labels", async () => { const html = await response.text(); assert.match(html, /🦞 ClawSweeper Live<\/title>/); assert.match(html, /content: "🦞"/); - assert.match(html, /Claw Workers/); + assert.match(html, /Codex Workers/); assert.match(html, /Active Sweeps/); assert.match(html, /Queue Depth/); assert.match(html, /Health Trends/); @@ -4678,6 +4733,16 @@ test("dashboard HTML preserves UTF-8 emoji labels", async () => { assert.match(html, /Error Rate/); assert.match(html, /Recovery Rate/); assert.match(html, /Capacity/); + assert.match(html, /Only jobs that execute Codex count against this budget/); + assert.match(html, /id="exact-review-lanes"/); + assert.match(html, /Review admission/); + assert.match(html, /Result publication/); + assert.match(html, /Control Plane · GitHub Actions, not Codex/); + assert.match(html, /id="control-plane"/); + assert.match(html, /\.exact-lanes \{ grid-template-columns: 1fr; \}/); + assert.ok(html.indexOf("Codex Capacity") < html.indexOf('id="exact-review-lanes"')); + assert.ok(html.indexOf('id="exact-review-lanes"') < html.indexOf('id="control-plane"')); + assert.ok(html.indexOf('id="control-plane"') < html.indexOf("Handoff Health")); assert.match(html, /Live terminals/); assert.match(html, /href="https:\/\/fleet\.example\.test\/terminal\?view=live&mode=all"/); assert.match(html, /Loading pipeline state/); @@ -4764,7 +4829,36 @@ test("dashboard hero treats apply and exact-review handoff health as attention", workers: [], automatic_work: [], pipeline: [], + control_plane: { + publishers: { running: 2, waiting: 1 }, + comment_routers: { running: 3, waiting: 4 }, + reconcilers: { running: 1, waiting: 0 }, + }, exact_review_queue: { + lanes: { + review: { + pending: 4, + ready: 3, + backoff: 1, + dispatching: 2, + leased: 10, + active: 12, + capacity: 64, + available_slots: 52, + oldest_pending_age_seconds: 60, + }, + publication: { + pending: 2, + ready: 1, + backoff: 1, + dispatching: 1, + leased: 20, + active: 21, + capacity: 24, + available_slots: 3, + oldest_pending_age_seconds: 30, + }, + }, handoff_health: { status: "healthy", message: "Dispatch-to-claim handoffs are within the expected window.", @@ -4875,6 +4969,18 @@ test("dashboard hero treats apply and exact-review handoff health as attention", assert.match(elementFor("exact-review-handoff").innerHTML, /Dispatching/); assert.match(elementFor("exact-review-handoff").innerHTML, /2 of 28 exact-review slots open/); assert.match(elementFor("exact-review-handoff").innerHTML, /health-badge healthy/); + assert.match(elementFor("exact-review-lanes").innerHTML, /Review admission/); + assert.match(elementFor("exact-review-lanes").innerHTML, /52 review admission slots open/); + assert.match(elementFor("exact-review-lanes").innerHTML, /Result publication/); + assert.match(elementFor("exact-review-lanes").innerHTML, /3 result publication slots open/); + assert.match(elementFor("control-plane").innerHTML, /2 running · 1 waiting/); + assert.match(elementFor("control-plane").innerHTML, /GitHub runners but not the 128-slot/); + + status.workers = Array.from({ length: 130 }, (_, id) => ({ id, status: "in_progress" })); + context.renderSystemMap(status); + assert.match(elementFor("capacity-rail").innerHTML, /130 running/); + assert.match(elementFor("capacity-rail").innerHTML, /2 over budget/); + status.workers = []; status.recent.apply_health.items = []; status.exact_review_queue.handoff_health.status = "stalled"; @@ -5307,6 +5413,82 @@ test("dashboard exposes active worker jobs and their current steps", async () => } }); +test("dashboard keeps control-plane workflow fallbacks out of Codex capacity", async () => { + const originalFetch = globalThis.fetch; + const originalCaches = globalThis.caches; + Object.defineProperty(globalThis, "caches", { + configurable: true, + value: { default: new MemoryCache() }, + }); + const runs = [ + [1, "ClawSweeper", "Review event item openclaw/openclaw#1", "in_progress"], + [2, "repair cluster worker", "repair cluster jobs/openclaw/inbox/cluster-2.md", "queued"], + [3, "Assist", "Assist openclaw/openclaw#3", "in_progress"], + [4, "ClawSweeper", "Review event item openclaw/openclaw#4@publish:40:1", "in_progress"], + [5, "repair comment router", "clawsweeper_comment", "queued"], + [6, "Reconcile exact-review leases", "Reconcile exact-review leases", "in_progress"], + [7, "ClawSweeper", "Sync Codex review comments for openclaw/openclaw", "queued"], + ].map(([id, name, displayTitle, status]) => ({ + id, + name, + display_title: displayTitle, + status, + conclusion: null, + html_url: `https://github.com/openclaw/clawsweeper/actions/runs/${id}`, + created_at: isoAgo(Number(id) * 1_000), + updated_at: isoAgo(500), + })); + globalThis.fetch = async (input) => { + const url = new URL(String(input)); + if (url.pathname === "/repos/openclaw/clawsweeper/actions/runs") { + const status = url.searchParams.get("status"); + return jsonResponse({ + workflow_runs: !status ? runs : runs.filter((run) => run.status === status), + }); + } + if (/^\/repos\/openclaw\/clawsweeper\/actions\/runs\/\d+\/jobs$/.test(url.pathname)) { + return jsonResponse({ jobs: [] }); + } + if ( + url.pathname === + "/repos/openclaw/clawsweeper/actions/workflows/repair-cluster-intake.yml/runs" + ) { + return jsonResponse({ workflow_runs: [] }); + } + if (url.pathname === "/search/issues") return jsonResponse({ items: [] }); + if (url.pathname === "/repos/openclaw/openclaw/issues") return jsonResponse([]); + throw new Error(`unexpected fetch ${url}`); + }; + + try { + const response = await worker.fetch( + new Request("https://clawsweeper.openclaw.ai/api/status"), + { + CLAWSWEEPER_REPO: "openclaw/clawsweeper", + TARGET_REPOS: "openclaw/openclaw", + CACHE_TTL_SECONDS: "0", + }, + { waitUntil: () => undefined }, + ); + const status = await response.json(); + assert.equal(status.fleet.active_codex_jobs, 3); + assert.equal(status.fleet.worker_detail_fallbacks, 3); + assert.deepEqual(status.workers.map((entry: { id: string }) => entry.id).sort(), [ + "run-1", + "run-2", + "run-3", + ]); + assert.deepEqual(status.control_plane, { + publishers: { running: 1, waiting: 0 }, + comment_routers: { running: 0, waiting: 2 }, + reconcilers: { running: 1, waiting: 0 }, + }); + } finally { + globalThis.fetch = originalFetch; + Object.defineProperty(globalThis, "caches", { configurable: true, value: originalCaches }); + } +}); + test("dashboard bounds worker job detail request concurrency", async () => { const originalFetch = globalThis.fetch; const originalCaches = globalThis.caches;