From fe6fa584a95df996025dd7a900bde35386a2948b Mon Sep 17 00:00:00 2001 From: brokemac79 Date: Tue, 21 Jul 2026 03:51:24 +0100 Subject: [PATCH 1/4] feat(dashboard): split Bay publishing lane --- dashboard/bay-page.ts | 19 ++-- dashboard/exact-review-queue.ts | 7 +- dashboard/worker.ts | 35 ++++++-- docs/proof/openclaw-bay/README.md | 2 +- docs/proof/openclaw-bay/run-proof.mjs | 32 ++++++- test/dashboard-worker.test.ts | 119 +++++++++++++++++++++++--- 6 files changed, 180 insertions(+), 34 deletions(-) diff --git a/dashboard/bay-page.ts b/dashboard/bay-page.ts index 31c3f33537..7affa71944 100644 --- a/dashboard/bay-page.ts +++ b/dashboard/bay-page.ts @@ -31,7 +31,7 @@ button,input,a{font:inherit} button{color:inherit} a{color:inherit} .repo-bar{margin-top:10px;display:flex;align-items:center;gap:7px;flex-wrap:wrap}.repo-bar>span{font:800 9px ui-monospace,monospace;color:var(--muted);text-transform:uppercase;letter-spacing:.12em}.repo-button{border:1px solid var(--line);background:rgba(255,255,255,.72);border-radius:999px;padding:6px 10px;font-size:10px;cursor:pointer}.repo-button:before{content:"";display:inline-block;width:8px;height:8px;border-radius:2px;background:var(--repo-color,#788);margin-right:6px}.repo-button.active{border-color:var(--repo-color,var(--claw));box-shadow:inset 0 0 0 1px var(--repo-color,var(--claw));background:white} .notice{display:none;padding:8px 16px;background:#fff5d9;border-bottom:1px solid #e4c57d;color:#6d5421;font-size:11px}.notice.show{display:block} .beach{position:relative;min-height:690px;overflow:hidden;background:linear-gradient(180deg,var(--sand-light),var(--sand));isolation:isolate}.beach:before{content:"";position:absolute;z-index:-3;right:-2%;top:0;bottom:0;width:20%;background:repeating-radial-gradient(ellipse at 0 50%,transparent 0 21px,rgba(220,250,249,.38) 22px 24px,transparent 25px 42px),linear-gradient(90deg,#86c0bf,var(--sea))}.beach:after{content:"";position:absolute;z-index:-2;left:0;top:0;bottom:0;width:6.5%;background:repeating-linear-gradient(90deg,#8a603d 0 3px,#b98b61 3px 18px,#7f5839 18px 20px);clip-path:polygon(0 0,100% 7%,77% 100%,0 100%)} -.beach-inner{position:relative;min-height:690px;padding:64px 20% 22px 7.5%}.stage-grid{display:grid;grid-template-columns:repeat(5,minmax(120px,1fr));height:600px}.stage{position:relative;border-right:1px dashed rgba(100,72,41,.22);min-width:0}.stage:last-child{border-right:0}.stage h2,.pool h2{position:absolute;z-index:5;left:50%;top:0;translate:-50% 0;margin:0;padding:9px 12px;border-radius:4px;background:#ad7c50;color:#352417;box-shadow:0 4px 0 #765032,0 8px 16px rgba(66,45,25,.13);font:800 9px ui-monospace,monospace;letter-spacing:.13em;text-transform:uppercase;white-space:nowrap}.stage h2 span,.pool h2 span{margin-left:6px;opacity:.68}.stage-body{position:absolute;inset:58px 5px 0}.empty{position:absolute;left:12px;right:12px;top:48%;text-align:center;color:rgba(87,70,49,.52);font-size:10px}.overflow-note{position:absolute;z-index:7;left:50%;bottom:5px;translate:-50% 0;padding:5px 8px;border:1px solid rgba(100,72,41,.2);border-radius:999px;background:rgba(255,249,235,.92);color:#6f5a42;font:700 8px ui-monospace,monospace;white-space:nowrap}.loading{position:absolute;inset:0;display:grid;place-items:center;color:#765f42;font:700 12px ui-monospace,monospace}.loading i{display:block;width:34px;height:34px;margin:0 auto 10px;border:3px solid rgba(117,89,52,.2);border-top-color:var(--claw);border-radius:50%;animation:spin .8s linear infinite} +.beach-inner{position:relative;min-height:690px;padding:64px 20% 22px 7.5%}.stage-grid{display:grid;grid-template-columns:repeat(6,minmax(120px,1fr));height:600px}.stage{position:relative;border-right:1px dashed rgba(100,72,41,.22);min-width:0}.stage:last-child{border-right:0}.stage h2,.pool h2{position:absolute;z-index:5;left:50%;top:0;translate:-50% 0;margin:0;padding:9px 12px;border-radius:4px;background:#ad7c50;color:#352417;box-shadow:0 4px 0 #765032,0 8px 16px rgba(66,45,25,.13);font:800 9px ui-monospace,monospace;letter-spacing:.13em;text-transform:uppercase;white-space:nowrap}.stage h2 span,.pool h2 span{margin-left:6px;opacity:.68}.stage-body{position:absolute;inset:58px 5px 0}.empty{position:absolute;left:12px;right:12px;top:48%;text-align:center;color:rgba(87,70,49,.52);font-size:10px}.overflow-note{position:absolute;z-index:7;left:50%;bottom:5px;translate:-50% 0;padding:5px 8px;border:1px solid rgba(100,72,41,.2);border-radius:999px;background:rgba(255,249,235,.92);color:#6f5a42;font:700 8px ui-monospace,monospace;white-space:nowrap}.loading{position:absolute;inset:0;display:grid;place-items:center;color:#765f42;font:700 12px ui-monospace,monospace}.loading i{display:block;width:34px;height:34px;margin:0 auto 10px;border:3px solid rgba(117,89,52,.2);border-top-color:var(--claw);border-radius:50%;animation:spin .8s linear infinite} .terminal-stack{position:absolute;right:2.8%;top:64px;bottom:24px;width:15.5%;display:flex;flex-direction:column;gap:11px}.pool{position:relative;min-height:130px;border:1px dashed rgba(94,66,37,.25);border-radius:50%;background:rgba(204,210,190,.52);overflow:visible}.pool.failed{background:rgba(236,190,157,.54);border-color:rgba(180,73,45,.45)}.pool.cancelled{background:rgba(206,199,179,.52)}.pool-body{position:absolute;inset:47px 8px 8px}.pool .empty{top:45%} .critter{position:absolute;left:var(--x);top:var(--y);width:76px;height:72px;border:0;background:transparent;padding:18px 3px 0;cursor:pointer;filter:drop-shadow(0 6px 4px rgba(76,53,29,.23));transition:transform .22s,filter .22s;animation:breathe 3.4s ease-in-out infinite;animation-delay:var(--delay)}.critter:hover,.critter:focus-visible{z-index:25;transform:translateY(-5px) scale(1.1);filter:drop-shadow(0 10px 7px rgba(76,53,29,.3));outline:none}.critter svg{width:67px;height:49px;overflow:visible;filter:hue-rotate(var(--hue)) saturate(var(--sat)) brightness(var(--light));transform:scale(var(--sx),var(--sy)) rotate(var(--tilt));transform-origin:center bottom}.critter .ref{position:absolute;z-index:4;left:50%;top:0;translate:-50% 0;white-space:nowrap;padding:4px 6px;border:1px solid var(--repo-color);border-radius:5px;background:rgba(255,255,255,.94);box-shadow:0 3px 8px rgba(60,45,28,.12);color:#51463a;font:700 8px ui-monospace,monospace}.repo-pin{position:absolute;right:3px;bottom:5px;width:7px;height:7px;border-radius:50%;background:var(--repo-color);border:1px solid white}.pool .critter{width:65px;height:60px;padding-top:16px}.pool .critter svg{width:54px;height:40px}.pool .critter .ref{font-size:7px}.critter.located{z-index:50;animation:found .7s ease-in-out infinite;filter:drop-shadow(0 0 5px white) drop-shadow(0 0 15px #f2a12d)}.find-arrow{display:none;position:absolute;z-index:6;left:50%;top:-45px;translate:-50% 0;padding:5px 7px;border:2px solid #96301f;border-radius:999px;background:var(--claw);color:white;box-shadow:0 4px 0 #96301f;font:900 8px ui-monospace,monospace;white-space:nowrap}.find-arrow:after{content:"↓";position:absolute;left:50%;top:20px;translate:-50%;color:var(--claw);font-size:25px}.critter.located .find-arrow{display:block}.critter.retriggered{animation:emerge 1.7s cubic-bezier(.2,.8,.2,1),breathe 3.4s 1.7s ease-in-out infinite}.critter.terminal-completed .claw{transform-origin:center;animation:wave-claw 1.5s ease-in-out infinite}.tide-imminent .terminal-completed .claw{animation-duration:.38s}.critter.terminal-failed{filter:drop-shadow(0 0 8px rgba(190,65,38,.4))}.critter.terminal-cancelled{filter:grayscale(.55) sepia(.2) drop-shadow(0 5px 4px rgba(76,53,29,.18))} .master{position:absolute;z-index:14;left:7%;top:50%;width:215px;height:100px;pointer-events:none;transition:left 3.4s cubic-bezier(.42,.04,.22,1),top 3.4s cubic-bezier(.42,.04,.22,1),transform .5s}.master svg{position:absolute;left:0;top:5px;width:135px;height:79px;filter:drop-shadow(0 7px 5px rgba(76,53,29,.25))}.brush{position:absolute;left:116px;top:49px;width:88px;height:9px;border-radius:8px;background:linear-gradient(#775032,#4e321f);transform:rotate(2deg);transform-origin:left center}.brush:after{content:"";position:absolute;right:-3px;top:-18px;width:27px;height:45px;border-radius:5px 10px 10px 5px;background:repeating-linear-gradient(90deg,#c59041 0 3px,#e5be6e 3px 6px);border:1px solid #98682d;transform:rotate(-3deg)}.master.sweeping .brush:after{animation:brush 1s ease-in-out infinite}.station{position:absolute;z-index:2;left:1.5%;top:43%;width:72px;height:155px;color:#206b70}.station strong{position:absolute;top:-22px;left:-3px;background:#28767a;color:white;border-radius:4px;padding:5px 7px;font:800 7px ui-monospace,monospace;text-transform:uppercase;white-space:nowrap}.station:before{content:"";position:absolute;left:17px;top:8px;width:42px;height:61px;border:5px solid #f4eee0;background:repeating-linear-gradient(0deg,#367f82 0 6px,#e9e3d6 6px 12px)}.station:after{content:"";position:absolute;left:9px;top:70px;width:58px;height:72px;border-left:5px solid #eee5d5;border-right:5px solid #eee5d5;clip-path:polygon(20% 0,80% 0,100% 100%,80% 100%,62% 28%,38% 28%,20% 100%,0 100%)} @@ -168,9 +168,9 @@ dialog{border:0;padding:0;margin:0 0 0 auto;width:min(580px,94vw);height:100vh;m diff --git a/dashboard/exact-review-queue.ts b/dashboard/exact-review-queue.ts index 0ecddf1208..b22fe65659 100644 --- a/dashboard/exact-review-queue.ts +++ b/dashboard/exact-review-queue.ts @@ -1920,7 +1920,11 @@ export class ExactReviewQueue { return json({ ...stats, pressure: elevateExactReviewPressureForPublication(stats.pressure, publicationHealth), - bay_projection: exactReviewQueueBayProjection(Object.values(state.items), bayPriorityKeys), + bay_projection: exactReviewQueueBayProjection( + Object.values(state.items), + bayPriorityKeys, + batchOwnedItemKeys, + ), lanes: { review: { ...stats.lanes.review, @@ -7285,7 +7289,14 @@ type ExactReviewBayProjectionItem = { next_attempt_at: string; }; -function exactReviewQueueBayStage(item: ExactReviewQueueItem): ExactReviewBayStage { +function exactReviewQueueBayStage( + item: ExactReviewQueueItem, + batchOwnedItemKeys: ReadonlySet = new Set(), +): ExactReviewBayStage { + // The batch publisher's GitHub job is intentionally targetless. Its durable + // batch membership is the authoritative bounded source for the individual + // items it is currently applying, without another GitHub lookup. + if (batchOwnedItemKeys.has(item.key)) return "applying"; if (exactReviewQueueIsPublication(item)) return "publishing"; if (isLowPriorityExactReviewDecision(item.decision)) return "repairing"; return item.state === "pending" ? "arriving" : "setting-up"; @@ -7309,6 +7320,7 @@ function exactReviewQueueBayPriorityKeys(values: string[]) { function exactReviewQueueBayProjection( items: ExactReviewQueueItem[], priorityItemKeys: string[] = [], + batchOwnedItemKeys: ReadonlySet = new Set(), ) { const projected = new Map(); for (const item of items) { @@ -7320,7 +7332,7 @@ function exactReviewQueueBayProjection( item_key: `${repository}#${itemNumber}`, repository, item_number: itemNumber, - stage: exactReviewQueueBayStage(item), + stage: exactReviewQueueBayStage(item, batchOwnedItemKeys), queue_state: item.state, created_at: new Date(item.createdAt).toISOString(), updated_at: new Date(item.updatedAt).toISOString(), diff --git a/dashboard/worker.ts b/dashboard/worker.ts index 9265529dec..0412309724 100644 --- a/dashboard/worker.ts +++ b/dashboard/worker.ts @@ -3255,15 +3255,15 @@ async function activeWorkerSnapshot( const detailRuns: WorkflowRunSummary[] = runs.slice(0, detailRunLimit); const results = await mapWithConcurrency(detailRuns, fetchConcurrency, async (run) => { try { - const jobs = await workflowJobsForRun(env, repo, run.id, github); + const jobs = await workflowJobsForRun(env, repo, run.id, github, run); const activeJobs = jobs.filter((job) => isActiveWorkflowJob(job)); return { run, workers: activeJobs - .filter((job) => isDashboardWorkerJob(job)) + .filter((job) => isDashboardWorkerJob(job, run)) .map((job) => normalizeWorkerJob(run, job)), codexWorkers: activeJobs.filter((job) => isCodexWorkerJob(job)).length, - hasWorkerJobs: jobs.some((job) => isDashboardWorkerJob(job)), + hasWorkerJobs: jobs.some((job) => isDashboardWorkerJob(job, run)), error: null, }; } catch (error) { @@ -3356,7 +3356,7 @@ async function recentWorkerHealth( const results = await mapWithConcurrency(completedRuns, fetchConcurrency, async (run) => { try { return { - attempts: (await workflowJobsForRun(env, repo, run.id, github)) + attempts: (await workflowJobsForRun(env, repo, run.id, github, run)) .filter((job) => isCodexWorkerJob(job)) .map((job) => workerHealthAttempt(run, job)) .filter(Boolean), @@ -4252,6 +4252,7 @@ async function workflowJobsForRun( repo, runId, github: GithubJsonReader = (path) => githubJson(env, path), + run?: WorkflowRunSummary, ) { const key = `workflow-jobs:${repo}:${runId}`; const cached = await readStoredJson(env, key); @@ -4271,7 +4272,9 @@ async function workflowJobsForRun( break; } } - const hasActiveWorker = jobs.some((job) => isActiveWorkflowJob(job) && isDashboardWorkerJob(job)); + const hasActiveWorker = jobs.some( + (job) => isActiveWorkflowJob(job) && isDashboardWorkerJob(job, run), + ); await writeStoredJson( env, key, @@ -4296,21 +4299,23 @@ function isCodexWorkerJob(job) { ); } -function isExactReviewPublicationJob(job) { +function isExactReviewPublicationJob(job, run?: WorkflowRunSummary) { const name = String(job?.name || ""); const steps = Array.isArray(job?.steps) ? job.steps : []; + const workflow = `${run?.name || ""} ${run?.display_title || ""}`; return ( - /publish exact review artifact/i.test(name) || + /publish (?:exact )?review artifacts?/i.test(name) || + (/publish exact review batch/i.test(workflow) && /^publish$/i.test(name)) || steps.some((step) => - /claim durable exact review publication|publish event result and apply safe close|complete durable exact review publication/i.test( + /claim durable exact review publication|claim one durable publication batch|finalize healthy members under a fenced heartbeat|publish event result and apply safe close|complete durable exact review publication|apply review artifacts|publish review artifact action ledger|commit review records/i.test( String(step?.name || ""), ), ) ); } -function isDashboardWorkerJob(job) { - return isCodexWorkerJob(job) || isExactReviewPublicationJob(job); +function isDashboardWorkerJob(job, run?: WorkflowRunSummary) { + return isCodexWorkerJob(job) || isExactReviewPublicationJob(job, run); } function normalizeWorkerJob(run, job) { @@ -4335,6 +4340,7 @@ function normalizeWorkerJob(run, job) { return { id: job.id, source: "job", + is_codex_worker: isCodexWorkerJob(job), name: String(job.name || runItem.title || "Codex worker"), mode, work_kind: workKind, @@ -4394,6 +4400,7 @@ function normalizeFallbackWorker(run) { return { id: `run-${run.id}`, source: "workflow-fallback", + is_codex_worker: true, name: item.title || item.workflow || "Codex worker", mode: item.mode, work_kind: workerWorkKind(item, ""), @@ -6237,7 +6244,7 @@ function workflowRunSummary(run) { function isCodexWorkflowFallback(run) { const name = `${run?.name || ""} ${run?.display_title || ""}`; if ( - /repair comment router|clawsweeper_comment|@publish:|publish exact review artifact|exact.review publication|reconcile exact.review lease|sync codex review comments/i.test( + /repair comment router|clawsweeper_comment|@publish:|publish (?:exact )?review (?:artifacts?|batch)|exact.review publication|reconcile exact.review lease|sync codex review comments/i.test( name, ) ) { @@ -6256,13 +6263,16 @@ function controlPlaneSnapshot(runs) { }; for (const run of runs) { const name = `${run?.name || ""} ${run?.display_title || ""}`; - const lane = /@publish:|publish exact review artifact|exact.review publication/i.test(name) - ? snapshot.publishers - : /repair comment router|clawsweeper_comment|sync codex review comments/i.test(name) - ? snapshot.comment_routers - : /reconcile exact.review lease/i.test(name) - ? snapshot.reconcilers - : null; + const lane = + /@publish:|publish (?:exact )?review (?:artifacts?|batch)|exact.review publication/i.test( + name, + ) + ? snapshot.publishers + : /repair comment router|clawsweeper_comment|sync codex review comments/i.test(name) + ? snapshot.comment_routers + : /reconcile exact.review lease/i.test(name) + ? snapshot.reconcilers + : null; if (!lane) continue; if (run.status === "in_progress") lane.running += 1; else lane.waiting += 1; @@ -9019,6 +9029,7 @@ function laneFlowDetails(laneKey, flow) { } function renderSystemMap(data) { const workers = data.workers || []; + const codexWorkers = workers.filter(worker => worker.is_codex_worker !== false); const pipeline = data.pipeline || []; const fleet = data.fleet || {}; const workerRunIds = new Set(workers.map(worker => String(worker.run_id))); @@ -9028,7 +9039,7 @@ function renderSystemMap(data) { const nodes = [ ["01 · Intake", fleet.queued_workflow_runs || 0, "Events and scheduled sweeps waiting to start"], ["02 · Plan", planning, "Runs selecting work or expanding a matrix"], - ["03 · Workers", workers.length, "Codex jobs reviewing, repairing, or assisting"], + ["03 · Workers", codexWorkers.length, "Codex jobs reviewing, repairing, or assisting"], ["04 · Apply", applying, "Deterministic comment, close, merge, and publish lanes"], ["05 · Results", closed, (data.recent?.closed_stats?.window_hours || 24) + "h ClawSweeper closes"] ]; @@ -9036,8 +9047,8 @@ function renderSystemMap(data) { '
' + esc(node[0]) + '' + fmt.format(node[1]) + '

' + esc(node[2]) + '

' ).join(""); const budget = Math.max(0, fleet.worker_budget || 0); - const running = workers.filter(worker => worker.status === "in_progress").length; - const waiting = workers.length - running; + const running = codexWorkers.filter(worker => worker.status === "in_progress").length; + const waiting = codexWorkers.length - running; const free = Math.max(0, budget - running - waiting); const overflow = Math.max(0, running + waiting - budget); const share = value => budget ? Math.min(100, (value / budget) * 100) : 0; @@ -9557,7 +9568,7 @@ function renderDashboard(data, note) { ); const severity = serverHealth?.severity || (handoffStatus === "stalled" || operationalStatus === "stalled" ? "red" : needsAttention ? "amber" : "green"); - const workerCount = (data.workers || []).length; + const workerCount = (data.workers || []).filter(worker => worker.is_codex_worker !== false).length; const repoCount = (data.source.target_repositories || []).length; document.getElementById("hero-dot").className = "hero-dot " + (severity === "green" ? "ok" : severity); document.getElementById("hero-headline").textContent = diff --git a/docs/proof/openclaw-bay/README.md b/docs/proof/openclaw-bay/README.md index 6785d02c2e..1cfe11ea09 100644 --- a/docs/proof/openclaw-bay/README.md +++ b/docs/proof/openclaw-bay/README.md @@ -18,7 +18,7 @@ The sequence proves: - repository filtering; - the read-only drawer's safe GitHub item, job, and workflow-run links; - readable overflow controls that open the known queue sample and explicitly explain when aggregate queue IDs are outside the bounded public projection; -- compact review-admission and result-publication charts with labelled y-axes, exact point hover labels, and cached 6-hour, 24-hour, and 7-day range controls; +- compact review-admission, result-publication, and State writer charts with labelled y-axes, exact point hover labels, and cached 6-hour, 24-hour, and 7-day range controls; - lightweight hover/focus explanations on the beach lane signs; - the local-only tide preview advancing through incoming, crest, backwash, and restored states while preserving terminal keys and count; - the short static reduced-motion tide cue preserving the same preview state; diff --git a/docs/proof/openclaw-bay/run-proof.mjs b/docs/proof/openclaw-bay/run-proof.mjs index 678a6fe60d..bdb6c7307b 100644 --- a/docs/proof/openclaw-bay/run-proof.mjs +++ b/docs/proof/openclaw-bay/run-proof.mjs @@ -247,9 +247,24 @@ let healthHistory = Array.from({ length: 73 }, (_, index) => { completed_total: 318 + index * 3, }, }, + state_writer: { + collection_ok: true, + mode: "batch", + tracked_holding: index % 5 === 0 ? 1 : 0, + tracked_waiting: 7 + (index % 4), + tracked_releasing: 0, + accepted_operations_total: 320 + index, + state_commits_total: 120 + index, + materialized_items_total: 360 + index * 2, + contention_timeouts_total: 0, + wait_ms: { p50: 1200, p95: 2800, samples: 4 }, + hold_ms: { p50: 400, p95: 900, samples: 4 }, + last_successful_materialization_at: at, + }, }; }); let healthHistoryFailure = false; +let stateWriterTerminalFresh = true; function queueProjection() { const bayStages = [ @@ -290,6 +305,13 @@ function queueProjection() { leased: { count: 33 }, }, }, + state_writer: { + collection: { status: "fresh" }, + mode: "batch", + live: { tracked_holding: 1, tracked_waiting: 8 }, + coordinator: { leased: 1, queued: 8 }, + last_60_minutes: { state_commits: 9, materialized_items: 17 }, + }, bay_projection: { sample_limit: 24, total: 36, @@ -420,6 +442,47 @@ const bayRetryLiveSnapshot = { }, }; proofSnapshots.push(bayRetryTerminalSnapshot, bayRetryLiveSnapshot); +const batchApplyingProjection = queueProjection(); +const batchApplyingItemKey = "openclaw/openclaw#108003"; +const batchApplyingItems = batchApplyingProjection.bay_projection.items.map((item) => + item.item_key === batchApplyingItemKey + ? { ...item, stage: "applying", queue_state: "pending" } + : item, +); +proofSnapshots.push({ + ...snapshots[0], + exact_review_queue: { + ...batchApplyingProjection, + bay_projection: { + ...batchApplyingProjection.bay_projection, + stages: { + ...batchApplyingProjection.bay_projection.stages, + publishing: batchApplyingProjection.bay_projection.stages.publishing - 1, + applying: batchApplyingProjection.bay_projection.stages.applying + 1, + }, + items: batchApplyingItems, + }, + }, +}); +const legacyApplyingProjection = queueProjection(); +const { publishing: legacyPublishingCount, ...legacyStages } = + legacyApplyingProjection.bay_projection.stages; +proofSnapshots.push({ + ...snapshots[0], + exact_review_queue: { + ...legacyApplyingProjection, + bay_projection: { + ...legacyApplyingProjection.bay_projection, + stages: { + ...legacyStages, + applying: legacyPublishingCount, + }, + items: legacyApplyingProjection.bay_projection.items.map((item) => + item.stage === "publishing" ? { ...item, stage: "applying" } : item, + ), + }, + }, +}); let fixtureIndex = 0; const requests = []; @@ -536,11 +599,15 @@ await page.route("**/*", async (route) => { const request = route.request(); const url = new URL(request.url()); if (url.pathname === "/api/status") { + const status = structuredClone(proofSnapshots[fixtureIndex]); + if (!stateWriterTerminalFresh) { + status.exact_review_queue.state_writer.collection.status = "stale"; + } await route.fulfill({ status: 200, contentType: "application/json; charset=utf-8", headers: { "cache-control": "no-store", "x-clawsweeper-cache": "synthetic-proof" }, - body: JSON.stringify(proofSnapshots[fixtureIndex]), + body: JSON.stringify(status), }); return; } @@ -679,10 +746,11 @@ try { lane_help: await page.locator('[data-stage="arriving"] .lane-help summary').count(), }; assertProof( - "Bay mirrors cached exact-review admission, publication, and handoff telemetry", - bayControl.cards === 3 && + "Bay mirrors cached exact-review admission, publication, state-writer, and handoff telemetry", + bayControl.cards === 4 && /Review admission/i.test(bayControl.review) && /Result publication/i.test(bayControl.review) && + /State writer/i.test(bayControl.review) && /Queue handoff/i.test(bayControl.review) && /waiting/.test(bayControl.waiting_hover_label) && /\/ hour/.test(bayControl.rate_hover_label) && @@ -700,6 +768,25 @@ try { bayControl.lane_help === 1, bayControl, ); + + stateWriterTerminalFresh = false; + await page.evaluate(() => window.__bayProofPoll()); + await page.waitForFunction(() => + /terminal metrics unavailable/.test( + document.querySelector("#bay-control-board")?.textContent || "", + ), + ); + const staleStateWriter = await page.locator("#bay-control-board").innerText(); + assertProof( + "Bay does not present stale State writer terminal totals as current", + /State writer/i.test(staleStateWriter) && + /unknown items/.test(staleStateWriter) && + /terminal metrics unavailable/.test(staleStateWriter) && + !/17 items/.test(staleStateWriter), + { state_writer: staleStateWriter }, + ); + stateWriterTerminalFresh = true; + await page.evaluate(() => window.__bayProofPoll()); const terminalPoolCounts = { completed: await page.locator('[data-stage="completed"] .critter').count(), attention: await page.locator(".pool.attention .critter").count(), @@ -729,7 +816,7 @@ try { await capture( "01a-mini-control-board", "Mini queue control board", - "Bay reuses the dashboard’s cached six-hour review-admission, publication, and handoff telemetry; each sparkline point has an exact hover label.", + "Bay reuses the dashboard’s cached six-hour review-admission, publication, state-writer, and handoff telemetry; each sparkline point has an exact hover label.", ); await page.locator('[data-stage="arriving"] .overflow-note').click(); @@ -883,7 +970,11 @@ try { const failedCollectionAt = originalHistory.at(-2)?.at; healthHistory = [ originalHistory.at(-3), - { at: failedCollectionAt, exact_review: { collection_ok: false } }, + { + at: failedCollectionAt, + exact_review: { collection_ok: false }, + state_writer: { collection_ok: false }, + }, originalHistory.at(-1), ]; const failedCollectionHistory = page.waitForResponse( @@ -910,6 +1001,52 @@ try { (failedCollectionPath?.match(/M/g) || []).length === 2, { review_pending_path: failedCollectionPath }, ); + const failedStateWriterPath = await page + .locator("#bay-control-board .bay-control-card") + .filter({ hasText: "State writer" }) + .locator(".bay-control-chart svg path") + .first() + .getAttribute("d"); + assertProof( + "Bay renders failed State writer history collections as a gap", + (failedStateWriterPath?.match(/M/g) || []).length === 2, + { state_writer_pending_path: failedStateWriterPath }, + ); + healthHistory = [ + originalHistory.at(-3), + originalHistory.at(-2), + { + at: originalHistory.at(-1)?.at, + exact_review: originalHistory.at(-1)?.exact_review, + state_writer: { collection_ok: false }, + }, + ]; + const currentStateWriterFailure = page.waitForResponse( + (response) => + new URL(response.url()).pathname === "/api/health-history" && response.status() === 200, + ); + await page.evaluate(() => { + window.__bayProofSetNow(Date.parse("2026-07-11T18:50:05.000Z")); + window.__bayProofPoll(); + }); + await currentStateWriterFailure; + await page.waitForFunction(() => + /history gap · awaiting current sample/i.test( + [...document.querySelectorAll("#bay-control-board .bay-control-card")] + .find((card) => /State writer/i.test(card.textContent || "")) + ?.textContent || "", + ), + ); + const failedStateWriterCopy = await page + .locator("#bay-control-board .bay-control-card") + .filter({ hasText: "State writer" }) + .innerText(); + assertProof( + "Bay labels a current failed State writer collection as a history gap", + /history gap · awaiting current sample/i.test(failedStateWriterCopy) && + !/history stale/i.test(failedStateWriterCopy), + { state_writer: failedStateWriterCopy }, + ); healthHistory = [ originalHistory.at(-3), @@ -928,7 +1065,7 @@ try { new URL(response.url()).pathname === "/api/health-history" && response.status() === 200, ); await page.evaluate(() => { - window.__bayProofSetNow(Date.parse("2026-07-11T18:49:04.000Z")); + window.__bayProofSetNow(Date.parse("2026-07-11T18:51:06.000Z")); window.__bayProofPoll(); }); await pendingOnlyHistory; @@ -949,6 +1086,41 @@ try { ); healthHistory = originalHistory; + await page.setViewportSize({ width: 900, height: 700 }); + await page.waitForFunction(() => + document.getElementById("stage-grid")?.classList.contains("portrait-stack"), + ); + await page.waitForTimeout(140); + const compactLandscapeLayout = await page.evaluate(() => { + const stages = Array.from(document.querySelectorAll("#stage-grid .stage")).map((stage) => { + const rect = stage.getBoundingClientRect(); + return { + left: Math.round(rect.left), + top: Math.round(rect.top), + bottom: Math.round(rect.bottom), + }; + }); + const terminal = document.getElementById("terminal-stack")?.getBoundingClientRect(); + return { + scroll_width: document.documentElement.scrollWidth, + viewport_width: window.innerWidth, + stages, + terminal_top: Math.round(terminal?.top || 0), + }; + }); + assertProof( + "intermediate landscape stacks six Bay lanes before they reach terminal pools", + compactLandscapeLayout.scroll_width <= compactLandscapeLayout.viewport_width + 1 && + compactLandscapeLayout.stages.length === 6 && + compactLandscapeLayout.stages.every((stage, index, stages) => + index === 0 + ? stage.left >= 0 + : Math.abs(stage.left - stages[0].left) <= 2 && stage.top >= stages[index - 1].bottom, + ) && + compactLandscapeLayout.terminal_top >= compactLandscapeLayout.stages.at(-1).bottom, + compactLandscapeLayout, + ); + await page.setViewportSize({ width: 390, height: 844 }); await page.waitForFunction(() => document.getElementById("stage-grid")?.classList.contains("portrait-stack"), @@ -965,10 +1137,12 @@ try { }); const terminal = document.getElementById("terminal-stack")?.getBoundingClientRect(); const grid = document.getElementById("stage-grid"); + const controlBoard = document.getElementById("bay-control-board"); return { scroll_width: document.documentElement.scrollWidth, viewport_width: window.innerWidth, grid_columns: getComputedStyle(grid).gridTemplateColumns, + control_columns: getComputedStyle(controlBoard).gridTemplateColumns, stages, terminal_top: Math.round(terminal?.top || 0), }; @@ -976,7 +1150,8 @@ try { assertProof( "portrait layout stacks the workflow from sand to waterline without horizontal overflow", portraitLayout.scroll_width <= portraitLayout.viewport_width + 1 && - portraitLayout.stages.length === 5 && + portraitLayout.stages.length === 6 && + portraitLayout.control_columns.trim().split(/\s+/).length === 1 && portraitLayout.stages.every((stage, index, stages) => index === 0 ? stage.left >= 0 @@ -1553,6 +1728,33 @@ try { "The stale failed card is absent and the same GitHub reference is visible as queued exact-review work in the earlier lane.", ); + fixtureIndex = 9; + await page.evaluate(async () => { + await window.__bayProofPoll(); + }); + const batchPublisherLane = page.locator(`[data-stage="applying"] [data-key="${batchApplyingItemKey}"]`); + await batchPublisherLane.waitFor({ state: "visible", timeout: 5_000 }); + assertProof( + "durable batch ownership places known publication items in Applying", + (await batchPublisherLane.count()) === 1, + { item_key: batchApplyingItemKey, stage: "applying" }, + ); + + fixtureIndex = 10; + await page.evaluate(async () => { + await window.__bayProofPoll(); + }); + const legacyQueuePublisher = page.locator( + `[data-stage="publishing"] [data-key="${batchApplyingItemKey}"]`, + ); + await legacyQueuePublisher.waitFor({ state: "visible", timeout: 5_000 }); + assertProof( + "legacy Applying queue records are normalized into Publishing without duplicate Applying cards", + (await legacyQueuePublisher.count()) === 1 && + (await page.locator('[data-stage="applying"] [data-item^="queue:"]').count()) === 0, + { item_key: batchApplyingItemKey, legacy_stage: "applying", rendered_stage: "publishing" }, + ); + const totalStatusGets = requests.filter((request) => request.path === "/api/status").length; const healthHistoryGets = requests.filter( (request) => request.path === "/api/health-history", @@ -1589,7 +1791,7 @@ try { }); assertProof( "mini control board caches each selected dashboard history range", - healthHistoryGets === 10 && + healthHistoryGets === 11 && healthHistoryRanges.filter((range) => range === "24h").length === 1 && healthHistoryRanges.filter((range) => range === "7d").length === 1, { health_history_gets: healthHistoryGets, ranges: healthHistoryRanges }, diff --git a/test/dashboard-worker.test.ts b/test/dashboard-worker.test.ts index feef1a8af9..20b397979b 100644 --- a/test/dashboard-worker.test.ts +++ b/test/dashboard-worker.test.ts @@ -2098,6 +2098,60 @@ test("dashboard status reads the exact-review handoff model from the durable que assert.equal(await exactReviewQueueStatusSnapshot({}), null); }); +test("Bay queue projection maps durable batch-owned publication items to Applying", async () => { + const storage = new MemoryDurableStorage(); + const queue = new ExactReviewQueue( + { storage }, + { + EXACT_REVIEW_DISPATCH_DEBOUNCE_MS: "0", + EXACT_REVIEW_PUBLICATION_BATCHING_ENABLED: "1", + EXACT_REVIEW_PUBLICATION_BATCH_SIZE: "1", + }, + ); + await queue.fetch( + buildExactReviewQueueRequest( + "bay-batch-applying", + 602, + "exact_review_artifact_publish", + "issue", + undefined, + exactReviewPublicationOverrides(602, "6020"), + ), + ); + const claim = await queue.fetch( + new Request("https://clawsweeper-exact-review-queue/publication-batches/claim", { + method: "POST", + body: JSON.stringify({ + claim_id: "bay-applying-batch", + lease_owner: "bay-applying-owner", + max_items: 1, + }), + }), + ); + assert.equal((await claim.json()).claimed, true); + + const status = await exactReviewQueueStatusSnapshot({ + EXACT_REVIEW_QUEUE: new MemoryDurableNamespace(queue), + }); + + assert.ok(status); + assert.deepEqual(status.bay_projection.stages, { + arriving: 0, + "setting-up": 0, + reviewing: 0, + publishing: 0, + applying: 1, + repairing: 0, + }); + assert.deepEqual( + status.bay_projection.items.map((item) => ({ + item_key: item.item_key, + stage: item.stage, + })), + [{ item_key: "openclaw/gogcli#602", stage: "applying" }], + ); +}); + test("Bay queue projection applies its public sample cap across all stages", async () => { const storage = new MemoryDurableStorage(); const queue = new ExactReviewQueue({ storage }, { EXACT_REVIEW_DISPATCH_DEBOUNCE_MS: "0" }); @@ -2269,9 +2323,13 @@ test("OpenClaw Bay is an unlisted, hardened demo route", async () => { assert.match(body, /id="bay-control-board"/); assert.match(body, /Review admission/); assert.match(body, /Result publication/); + assert.match(body, /State writer/); assert.match(body, /Queue handoff/); assert.match(body, /function loadBayHistory/); assert.match(body, /function bayRateSparkline/); + assert.match(body, /function bayStateWriterCard/); + assert.match(body, /function bayStateWriterHistory/); + assert.match(body, /max-width:970px\) and \(orientation:landscape/); assert.match(body, /net throughput over .*bayRangeLabel/); assert.match(body, /data-bay-history-range="24h"/); assert.match(body, /data-bay-history-range="7d"/); @@ -2388,12 +2446,13 @@ test("OpenClaw Bay is an unlisted, hardened demo route", async () => { ).runInNewContext({ STAGES: ["arriving", "setting-up", "reviewing", "publishing", "applying", "repairing"], }); - assert.equal(normalizeQueueStage("applying"), "publishing"); + assert.equal(normalizeQueueStage("applying", false), "applying"); + assert.equal(normalizeQueueStage("applying", true), "publishing"); assert.equal(normalizeQueueStage("publishing"), "publishing"); const classifyBayStage = new Script(`${stageForSource};stageFor`).runInNewContext({ STAGES: ["arriving", "setting-up", "reviewing", "publishing", "applying", "repairing"], }); - const publicationSteps = [{ name: "Claim durable exact review publication" }]; + const publicationSteps = [{ name: "Apply review artifacts" }]; assert.equal( classifyBayStage({ status: "in_progress", @@ -2412,7 +2471,7 @@ test("OpenClaw Bay is an unlisted, hardened demo route", async () => { ); assert.equal( classifyBayStage({ - name: "Publish exact review artifact", + name: "Publish review artifacts", status: "queued", current_step: "Waiting for runner", steps: [], @@ -2422,7 +2481,15 @@ test("OpenClaw Bay is an unlisted, hardened demo route", async () => { assert.equal( classifyBayStage({ status: "in_progress", - current_step: "Publish event result and apply safe close", + current_step: "Publish review artifact action ledger", + steps: publicationSteps, + }), + "publishing", + ); + assert.equal( + classifyBayStage({ + status: "in_progress", + current_step: "Sync selected review comments", steps: publicationSteps, }), "applying", @@ -10653,10 +10720,21 @@ test("dashboard hero treats apply and exact-review handoff health as attention", /target 32 · pressure ceiling 24 after GitHub rate limit/, ); assert.match(elementFor("exact-review-lanes").innerHTML, /No backlog history in this range/); - status.workers = Array.from({ length: 130 }, (_, id) => ({ id, status: "in_progress" })); + status.workers = [ + ...Array.from({ length: 128 }, (_, id) => ({ + id, + status: "in_progress", + is_codex_worker: true, + })), + { id: 128, status: "in_progress", is_codex_worker: false }, + { id: 129, status: "in_progress", is_codex_worker: false }, + ]; context.renderSystemMap(status); - assert.match(elementFor("capacity-rail").innerHTML, /130 running/); - assert.match(elementFor("capacity-rail").innerHTML, /2 over budget/); + assert.match(elementFor("capacity-rail").innerHTML, /128 running/); + assert.doesNotMatch(elementFor("capacity-rail").innerHTML, /over budget/); + status.workers = [{ id: 130, status: "in_progress", is_codex_worker: false }]; + context.renderDashboard(status, ""); + assert.match(elementFor("hero-headline").textContent, /0 claw workers sweeping/); status.workers = []; status.recent.apply_health.items = []; @@ -11118,6 +11196,16 @@ test("dashboard exposes active worker jobs and their current steps", async () => created_at: isoAgo(30_000), updated_at: isoAgo(5_000), }; + const queuedBatchRun = { + id: 44, + name: "Publish exact review batch", + display_title: "Publish exact review batch", + status: "queued", + conclusion: null, + html_url: "https://github.com/openclaw/clawsweeper/actions/runs/44", + created_at: isoAgo(20_000), + updated_at: isoAgo(2_000), + }; let graphqlRequests = 0; globalThis.fetch = async (input) => { const url = new URL(String(input)); @@ -11125,11 +11213,11 @@ test("dashboard exposes active worker jobs and their current steps", async () => const status = url.searchParams.get("status"); return jsonResponse({ workflow_runs: !status - ? [run, queuedRun] + ? [run, queuedRun, queuedBatchRun] : status === "in_progress" ? [run] : status === "queued" - ? [queuedRun] + ? [queuedRun, queuedBatchRun] : [], }); } @@ -11166,7 +11254,7 @@ test("dashboard exposes active worker jobs and their current steps", async () => }, { id: 4202, - name: "Publish exact review artifact", + name: "Publish review artifacts", status: "in_progress", conclusion: null, html_url: "https://github.com/openclaw/clawsweeper/actions/runs/42/job/4202", @@ -11174,13 +11262,35 @@ test("dashboard exposes active worker jobs and their current steps", async () => steps: [ { number: 1, - name: "Claim durable exact review publication", + name: "Apply review artifacts", status: "completed", conclusion: "success", }, { number: 2, - name: "Publish event result and apply safe close", + name: "Publish review artifact action ledger", + status: "in_progress", + conclusion: null, + }, + ], + }, + { + id: 4203, + name: "publish", + status: "in_progress", + conclusion: null, + html_url: "https://github.com/openclaw/clawsweeper/actions/runs/42/job/4203", + started_at: isoAgo(55_000), + steps: [ + { + number: 1, + name: "Claim one durable publication batch", + status: "completed", + conclusion: "success", + }, + { + number: 2, + name: "Finalize healthy members under a fenced heartbeat", status: "in_progress", conclusion: null, }, @@ -11192,6 +11302,21 @@ test("dashboard exposes active worker jobs and their current steps", async () => if (url.pathname === "/repos/openclaw/clawsweeper/actions/runs/43/jobs") { return jsonResponse({ jobs: [] }); } + if (url.pathname === "/repos/openclaw/clawsweeper/actions/runs/44/jobs") { + return jsonResponse({ + jobs: [ + { + id: 4401, + name: "publish", + status: "queued", + conclusion: null, + html_url: "https://github.com/openclaw/clawsweeper/actions/runs/44/job/4401", + started_at: null, + steps: [], + }, + ], + }); + } if (url.pathname === "/graphql") { graphqlRequests += 1; return jsonResponse({ @@ -11248,9 +11373,9 @@ test("dashboard exposes active worker jobs and their current steps", async () => ); const status = await response.json(); assert.equal(status.fleet.active_codex_jobs, 2); - assert.equal(status.fleet.worker_detail_runs, 2); + assert.equal(status.fleet.worker_detail_runs, 3); assert.equal(status.fleet.worker_detail_fallbacks, 1); - assert.equal(status.workers.length, 3); + assert.equal(status.workers.length, 5); assert.equal(status.workers[0].id, 4201); assert.equal(status.workers[0].name, "Review shard 0 · openclaw/openclaw#92521,92522"); assert.equal(status.workers[0].repository, "openclaw/openclaw"); @@ -11276,18 +11401,32 @@ test("dashboard exposes active worker jobs and their current steps", async () => }, ]); assert.equal(status.workers[1].id, 4202); - assert.equal(status.workers[1].name, "Publish exact review artifact"); + assert.equal(status.workers[1].name, "Publish review artifacts"); + assert.equal(status.workers[1].is_codex_worker, false); assert.equal(status.workers[1].item_number, 92521); - assert.equal(status.workers[1].current_step, "Publish event result and apply safe close"); - assert.equal(status.workers[1].steps[0].name, "Claim durable exact review publication"); + assert.equal(status.workers[1].current_step, "Publish review artifact action ledger"); + assert.equal(status.workers[1].steps[0].name, "Apply review artifacts"); + assert.equal(status.workers[2].id, 4203); + assert.equal(status.workers[2].name, "publish"); + assert.equal(status.workers[2].is_codex_worker, false); + assert.equal( + status.workers[2].current_step, + "Finalize healthy members under a fenced heartbeat", + ); const cachedPublisherJobs = await cache.match( new Request("https://clawsweeper.internal/store/workflow-jobs%3Aopenclaw%2Fclawsweeper%3A42"), ); assert.equal(cachedPublisherJobs?.headers.get("cache-control"), "public, max-age=60"); - assert.equal(status.workers[2].id, "run-43"); - assert.equal(status.workers[2].source, "workflow-fallback"); - assert.equal(status.workers[2].current_step, "reviewing"); - assert.equal(status.workers[2].target_items[0].title, "Queued terminal resize follow-up"); + assert.equal(status.workers[3].id, "run-43"); + assert.equal(status.workers[3].source, "workflow-fallback"); + assert.equal(status.workers[3].current_step, "reviewing"); + assert.equal(status.workers[3].target_items[0].title, "Queued terminal resize follow-up"); + const queuedBatchPublisher = status.workers.find((entry) => entry.id === 4401); + assert.ok(queuedBatchPublisher); + assert.equal(queuedBatchPublisher.name, "publish"); + assert.equal(queuedBatchPublisher.is_codex_worker, false); + assert.equal(queuedBatchPublisher.workflow_title, "Publish exact review batch"); + assert.equal(queuedBatchPublisher.current_step, "Waiting for runner"); const cachedResponse = await worker.fetch( new Request("https://clawsweeper.openclaw.ai/api/status"), From cccefa42b0847a927ef99527eb2b7cbfc82d6f24 Mon Sep 17 00:00:00 2001 From: brokemac79 Date: Fri, 24 Jul 2026 23:53:58 +0100 Subject: [PATCH 3/4] fix(dashboard): show parked Bay recovery work --- dashboard/exact-review-queue.ts | 9 ++++++- test/dashboard-worker.test.ts | 45 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/dashboard/exact-review-queue.ts b/dashboard/exact-review-queue.ts index b22fe65659..d54a5370bf 100644 --- a/dashboard/exact-review-queue.ts +++ b/dashboard/exact-review-queue.ts @@ -7293,6 +7293,11 @@ function exactReviewQueueBayStage( item: ExactReviewQueueItem, batchOwnedItemKeys: ReadonlySet = new Set(), ): ExactReviewBayStage { + // A parked item is deliberately no longer making normal queue progress. This + // includes bounded review-retry exhaustion, permanent dispatch rejection, + // and a publication that needs its dead-letter/recovery path. Keep it in the + // exception cove instead of making it look like an active setup or publisher. + if (item.state === "parked") return "repairing"; // The batch publisher's GitHub job is intentionally targetless. Its durable // batch membership is the authoritative bounded source for the individual // items it is currently applying, without another GitHub lookup. @@ -7324,7 +7329,9 @@ function exactReviewQueueBayProjection( ) { const projected = new Map(); for (const item of items) { - if (item.state === "parked") continue; + // Parked records are not terminal outcomes: they remain bounded durable + // queue work that needs recovery. Keep their already-scrubbed identity in + // the projection so Bay shows the exception rather than a false empty lane. const repository = String(item.decision.targetRepo || "").trim(); const itemNumber = Number(item.decision.itemNumber); if (!repository || !Number.isSafeInteger(itemNumber) || itemNumber <= 0) continue; diff --git a/test/dashboard-worker.test.ts b/test/dashboard-worker.test.ts index 20b397979b..feadf30a83 100644 --- a/test/dashboard-worker.test.ts +++ b/test/dashboard-worker.test.ts @@ -2152,6 +2152,51 @@ test("Bay queue projection maps durable batch-owned publication items to Applyin ); }); +test("Bay queue projection sends parked review and publication work to Repair Cove", async () => { + const storage = new MemoryDurableStorage(); + const queue = new ExactReviewQueue({ storage }, { EXACT_REVIEW_DISPATCH_DEBOUNCE_MS: "0" }); + await queue.fetch(buildExactReviewQueueRequest("bay-parked-review", 603, "opened")); + await queue.fetch( + buildExactReviewQueueRequest( + "bay-parked-publication", + 604, + "exact_review_artifact_publish", + "issue", + undefined, + exactReviewPublicationOverrides(604, "6040"), + ), + ); + const state = (await storage.get("exact-review-queue")) as { + items: Record; + }; + state.items["openclaw/gogcli#603"].state = "parked"; + state.items["openclaw/gogcli#603"].parkedReason = "review_retry_exhausted"; + state.items["openclaw/gogcli#604@publish:6040:1"].state = "parked"; + state.items["openclaw/gogcli#604@publish:6040:1"].parkedReason = "dead_letter_capacity"; + await storage.put("exact-review-queue", state); + + const status = await exactReviewQueueStatusSnapshot({ + EXACT_REVIEW_QUEUE: new MemoryDurableNamespace(queue), + }); + + assert.ok(status); + assert.deepEqual(status.bay_projection.stages, { + arriving: 0, + "setting-up": 0, + reviewing: 0, + publishing: 0, + applying: 0, + repairing: 2, + }); + assert.deepEqual( + status.bay_projection.items.map((item) => ({ item_key: item.item_key, stage: item.stage })), + [ + { item_key: "openclaw/gogcli#603", stage: "repairing" }, + { item_key: "openclaw/gogcli#604", stage: "repairing" }, + ], + ); +}); + test("Bay queue projection applies its public sample cap across all stages", async () => { const storage = new MemoryDurableStorage(); const queue = new ExactReviewQueue({ storage }, { EXACT_REVIEW_DISPATCH_DEBOUNCE_MS: "0" }); From b211a730268e04fa9373886ce7b5984dea86909c Mon Sep 17 00:00:00 2001 From: brokemac79 Date: Sat, 25 Jul 2026 14:34:04 +0100 Subject: [PATCH 4/4] feat(dashboard): show Bay publication batch ownership --- dashboard/bay-page.ts | 9 +++--- dashboard/exact-review-publication-batches.ts | 9 ++++++ dashboard/exact-review-queue.ts | 27 ++++++++++++++---- docs/proof/openclaw-bay/run-proof.mjs | 28 +++++++++++++------ test/dashboard-worker.test.ts | 9 +++++- test/exact-review-publication-batches.test.ts | 14 ++++++++++ 6 files changed, 77 insertions(+), 19 deletions(-) diff --git a/dashboard/bay-page.ts b/dashboard/bay-page.ts index 26f4c93999..c63dcae5a2 100644 --- a/dashboard/bay-page.ts +++ b/dashboard/bay-page.ts @@ -198,7 +198,7 @@ dialog{border:0;padding:0;margin:0 0 0 auto;width:min(580px,94vw);height:100vh;m "use strict"; var STAGES=["arriving","setting-up","reviewing","publishing","applying","repairing"]; var MAIN_STAGES=["arriving","setting-up","reviewing","publishing","applying"]; - var LABELS={"arriving":"Arriving","setting-up":"Setting up","reviewing":"Reviewing","publishing":"Publishing","repairing":"Repair cove","applying":"Applying","completed":"Completed","failed":"Failed · rescue pool","cancelled":"Cancelled"}; + var LABELS={"arriving":"Arriving","setting-up":"Setting up","reviewing":"Reviewing","publishing":"Publishing","repairing":"Repair cove","applying":"Applying & writing","completed":"Completed","failed":"Failed · rescue pool","cancelled":"Cancelled"}; var TERMINAL_GROUPS=[{id:"completed",label:"Completed",stages:["completed"]},{id:"attention",label:"Failed / cancelled",stages:["failed","cancelled"]}]; var OUTCOME_CONFIRM_MS=150000; var TIDE_CREST_MS=3500,TIDE_RECEDING_MS=4550,TIDE_REVEAL_MS=6100,TIDE_END_MS=7200; @@ -211,14 +211,15 @@ dialog{border:0;padding:0;margin:0 0 0 auto;width:min(580px,94vw);height:100vh;m function repoInfo(repo,index){var full=repo||"Other waters";var name=(full.split("/")[1]||"").toLowerCase();var known={"openclaw":"OC","clawhub":"CH","clawsweeper":"CS"};var short=known[name]||(name?name.replace(/[^a-z0-9]/g,"").slice(0,2).toUpperCase():"OT");return {label:full,short:short,color:repoPalette[index%repoPalette.length]};} function stageFor(worker){var status=String(worker.status||"").toLowerCase();var step=String(worker.current_step||"").trim().toLowerCase();var fallback=[worker.stage,worker.mode,worker.name].join(" ").toLowerCase();var stepHistory=(worker.steps||[]).map(function(item){return item&&item.name||"";}).join(" ").toLowerCase();var publishingRun=/publish (?:exact )?review (?:artifacts?|batch)|claim durable exact review publication|claim one durable publication batch|finalize healthy members under a fenced heartbeat|apply review artifacts|publish review artifact action ledger|commit review records|publish event result and apply safe close|complete durable exact review publication/.test([worker.name,worker.workflow_title,stepHistory].join(" ").toLowerCase());if(publishingRun){if(/finalize healthy members under a fenced heartbeat|sync selected review comments|dispatch selected safe close proposals to isolated apply|continue sweep|publish event result and apply safe close|queue deferred exact verdict router|queue fresh review after source drift|release terminal review leases|confirm terminal item remains closed|mark re-review complete|react to target item completion|release superseded or unsuccessful publisher-owned review lease|complete durable exact review publication|mark active lease retry waiting|fail unsuccessful exact review publication/.test(step))return "applying";return "publishing";}if(["queued","waiting","requested","pending"].indexOf(status)>=0)return "arriving";if(/setup-state|state token|publish|route|queue deferred|release|confirm terminal|mark .* complete|commit .* ledger|retry waiting|react .* completion|fail unsuccessful|complete .* lease/.test(step))return "applying";if(/set up job|setup-pnpm|setup-codex|checkout|check out|cache|claim .* lease|resolve .* payload|create target .* token|check live target|mark .* in progress|react .* review start/.test(step))return "setting-up";if(/review|codex|inspect|analyse|analyze|assist/.test(step))return "reviewing";if(/repair|fix|rebase|validat|test/.test(step))return "repairing";if(/apply|publish|merge|close|comment|route|release|complete/.test(step))return "applying";if(STAGES.indexOf(worker.stage)>=0)return worker.stage;if(/repair|fix|rebase/.test(fallback))return "repairing";if(/apply|publish|merge|close|automerge/.test(fallback))return "applying";if(/review|codex|assist/.test(fallback))return "reviewing";return "arriving";} function kindFor(item){var text=[item.work_kind,item.mode,item.name,item.title].join(" ").toLowerCase();if(/repair|fix/.test(text))return "crab";if(/issue.to.pr|build|implement/.test(text))return "hermit";if(/commit|background/.test(text))return "shrimp";return ["lobster","crab","shrimp","hermit"][hash(item.key)%4];} - function queueStageLabel(stage){return {"arriving":"Waiting for review admission","setting-up":"Setting up the review run","publishing":"Waiting to publish the final review","applying":"Applying deterministic follow-up","repairing":"Recovering the review"}[stage]||"Queued ClawSweeper work";} + function batchLabel(value){var text=String(value||"");var match=text.match(/:(\d+)$/);return match?"batch #"+match[1]:text?"publication batch":"the publication batch";} + function queueStageLabel(stage,source){if(stage==="applying"){var batch=source&&source.batch_id;return batch?"Applying & writing · "+batchLabel(batch)+" owns this item":"Applying & writing · final acknowledgement pending";}return {"arriving":"Waiting for review admission","setting-up":"Setting up the review run","publishing":"Waiting to publish the final review","repairing":"Recovering the review"}[stage]||"Queued ClawSweeper work";} function queueProjectionStage(stage,legacyApplying){if(legacyApplying&&stage==="applying")return "publishing";return STAGES.indexOf(stage)>=0?stage:"arriving";} - function expandQueue(data){var projection=data.exact_review_queue&&data.exact_review_queue.bay_projection;var rows=projection&&Array.isArray(projection.items)?projection.items:[];var legacyApplying=projection&&projection.stages&&!Object.prototype.hasOwnProperty.call(projection.stages,"publishing");return rows.map(function(item){var repository=String(item.repository||"unknown");var number=Number(item.item_number);if(!number)return null;var stage=queueProjectionStage(item.stage,legacyApplying);return {id:"queue:"+String(item.item_key||repository+"#"+number),key:String(item.item_key||repository+"#"+number),number:number,repository:repository,title:"Queued ClawSweeper work",item_url:"https://github.com/"+repository+"/issues/"+number,stage:stage,status:item.queue_state||"pending",outcome:null,run_id:null,current_step:queueStageLabel(stage),steps:[],progress:{},started_at:item.created_at,work_kind:"exact-review",mode:"queue",queue_item:true,source:item};}).filter(Boolean);} + function expandQueue(data){var projection=data.exact_review_queue&&data.exact_review_queue.bay_projection;var rows=projection&&Array.isArray(projection.items)?projection.items:[];var legacyApplying=projection&&projection.stages&&!Object.prototype.hasOwnProperty.call(projection.stages,"publishing");return rows.map(function(item){var repository=String(item.repository||"unknown");var number=Number(item.item_number);if(!number)return null;var stage=queueProjectionStage(item.stage,legacyApplying);return {id:"queue:"+String(item.item_key||repository+"#"+number),key:String(item.item_key||repository+"#"+number),number:number,repository:repository,title:"Queued ClawSweeper work",item_url:"https://github.com/"+repository+"/issues/"+number,stage:stage,status:item.queue_state||"pending",outcome:null,run_id:null,current_step:queueStageLabel(stage,item),steps:[],progress:{},started_at:item.created_at,work_kind:"exact-review",mode:"queue",queue_item:true,batch_id:item.batch_id||null,source:item};}).filter(Boolean);} // Keep the full, bounded queue total visible while letting a live worker replace its // known queue record. The durable queue only exposes a scrubbed sample of keys, so // subtract just those positively identified overrides and label unseen queue IDs separately. function queueStageSummary(stage,rows){var limit=densityFor(stage,rows.length).limit;var shown=rows.slice(0,limit);var projection=state.data&&state.data.exact_review_queue&&state.data.exact_review_queue.bay_projection;var queueRows=rows.filter(function(item){return item.queue_item;});var activeRows=rows.filter(function(item){return !item.queue_item;});var shownQueue=shown.filter(function(item){return item.queue_item;}).length;var shownActive=shown.filter(function(item){return !item.queue_item;}).length;if(state.filter!=="all"||!projection)return {label:String(rows.length),queueHidden:Math.max(0,queueRows.length-shownQueue),liveHidden:Math.max(0,activeRows.length-shownActive),projected:false};var activeKeys={};state.items.forEach(function(item){if(item&&!item.queue_item&&!item.outcome)activeKeys[item.key]=true;});var stages=projection.stages||{};var hasPublishing=Object.prototype.hasOwnProperty.call(stages,"publishing");var sample=(Array.isArray(projection.items)?projection.items:[]).filter(function(item){return item&&queueProjectionStage(item.stage,!hasPublishing)===stage;});var superseded=sample.filter(function(item){return activeKeys[String(item.item_key||"")];}).length;var reported=Math.max(0,Number(stages[stage])||0);if(stage==="publishing"&&!hasPublishing)reported=Math.max(0,Number(stages.applying)||0);if(stage==="applying"&&!hasPublishing)reported=0;var queued=Math.max(queueRows.length,reported-superseded);return {label:String(queued+activeRows.length),queueHidden:Math.max(0,queued-shownQueue),liveHidden:Math.max(0,activeRows.length-shownActive),projected:true};} - function laneHelp(id){var copy={"arriving":"New review requests waiting for admission. A queue item can move straight to a later lane when its live run has already begun.","setting-up":"A review run is claiming its work and preparing the repository, permissions, and tools it needs.","reviewing":"Codex is inspecting the exact GitHub item. This is the main review work, not necessarily a separate workflow run.","publishing":"The completed review artifact is waiting to be checked and posted back to its GitHub item. This is the bounded result-publication queue.","applying":"The publisher is applying deterministic follow-up such as a safe close, routing, or final confirmation. Successful work then reaches the completed pool.","repairing":"Exception path for a repair, validation, or retry. Most normal review journeys do not visit this cove.","completed":"Explicitly proved successful outcomes waiting for the next tide.","attention":"Explicitly proved failed or cancelled outcomes. They stay separate from success and wait for the same tide."};var text=copy[id];return text?'
i'+esc(text)+'
':"";} + function laneHelp(id){var copy={"arriving":"New review requests waiting for admission. A queue item can move straight to a later lane when its live run has already begun.","setting-up":"A review run is claiming its work and preparing the repository, permissions, and tools it needs.","reviewing":"Codex is inspecting the exact GitHub item. This is the main review work, not necessarily a separate workflow run.","publishing":"The completed review artifact is waiting to be checked and posted back to its GitHub item. This is the bounded result-publication queue.","applying":"A durable publication batch owns this item while it writes shared state and completes required follow-up. It stays here until the publisher acknowledges the batch; separately routed GitHub follow-up is shown as dispatched, not assumed complete.","repairing":"Exception path for a repair, validation, or retry. Most normal review journeys do not visit this cove.","completed":"Explicitly proved successful outcomes waiting for the next tide.","attention":"Explicitly proved failed or cancelled outcomes. They stay separate from success and wait for the same tide."};var text=copy[id];return text?'
i'+esc(text)+'
':"";} function laneHeading(id,label,count){return '

'+esc(label)+' '+esc(count)+''+laneHelp(id)+'

';} function overflowNote(stage,summary,terminal){var parts=[];if(summary.queueHidden)parts.push("+"+summary.queueHidden+" queued ID"+(summary.queueHidden===1?"":"s")+" not shown");if(summary.liveHidden)parts.push("+"+summary.liveHidden+" more live item"+(summary.liveHidden===1?"":"s")+" not shown");if(!parts.length)return "";var label=(terminal?"Open tide buffer details: ":"Open queue sample details: ")+parts.join(". ");return '';} function openQueueSampleDrawer(stage,terminal){var drawer=document.getElementById("queue-sample-drawer");var body=document.getElementById("queue-sample-body");if(!drawer||!body)return;var rows=visible(state.items).filter(function(item){return terminal?item.outcome&&((stage==="attention"&&["failed","cancelled"].indexOf(item.stage)>=0)||item.stage===stage):item.stage===stage;});var summary=terminal?{queueHidden:0,liveHidden:Math.max(0,rows.length-densityFor(stage,rows.length).limit),projected:false}:queueStageSummary(stage,rows);var projection=state.data&&state.data.exact_review_queue&&state.data.exact_review_queue.bay_projection||{};var sampleLimit=Number(projection.sample_limit||24);var title=terminal?(stage==="attention"?"Failed / cancelled tide buffer":LABELS[stage]+" tide buffer"):(LABELS[stage]+" queue sample");document.getElementById("queue-sample-title").textContent=title;document.getElementById("queue-sample-subtitle").textContent=terminal?"Known terminal outcomes held until the next tide.":"Known references from the bounded public queue projection.";var copy="";if(summary.projected&&summary.queueHidden)copy+='

'+summary.queueHidden+' queued ID'+(summary.queueHidden===1?" is":"s are")+' counted in this lane, but their individual references are intentionally outside OpenClaw Bay\'s '+sampleLimit+'-reference public sample. This panel does not fetch or invent that missing list.

';else if(summary.liveHidden)copy+='

The beach is showing its densest safe layout. All known references below remain available here rather than being squeezed into unreadable labels.

';else copy+='

These are the currently known references for this part of the beach.

';var known=rows;var list=known.length?'
    '+known.map(function(item){var href=safeUrl(item.item_url);var label=repoInfo(item.repository,0).short+" #"+item.number;return '
  • '+(href?''+esc(label)+'':''+esc(label)+'')+''+esc(item.current_step||item.status||"Queued ClawSweeper work")+'
  • ';}).join("")+"
":'

No individual references are available for the current filter.

';body.innerHTML=copy+list;if(!drawer.open)drawer.showModal();} diff --git a/dashboard/exact-review-publication-batches.ts b/dashboard/exact-review-publication-batches.ts index b962f23414..2a2936528f 100644 --- a/dashboard/exact-review-publication-batches.ts +++ b/dashboard/exact-review-publication-batches.ts @@ -56,6 +56,10 @@ export type PublicationBatchStats = { expired: number; activeItems: number; activeItemKeys: string[]; + // This contains only unfinished, currently leased membership. It lets the + // read-only Bay projection identify the bounded batch that owns an item + // without retaining a separate event history or looking it up through GitHub. + activeItemBatches: Array<{ itemKey: string; batchId: string }>; nextLeaseExpiresAt: number | null; oldestActiveAt: number | null; reclaimedItemsRetained: number; @@ -424,6 +428,7 @@ export class ExactReviewPublicationBatchStore { expired: counts.get("expired") ?? 0, activeItems: activeLease.itemKeys.length, activeItemKeys: activeLease.itemKeys, + activeItemBatches: activeLease.items, nextLeaseExpiresAt: activeLease.nextLeaseExpiresAt, oldestActiveAt: leased ? Number(leased.oldest_at) : null, reclaimedItemsRetained, @@ -470,6 +475,10 @@ export class ExactReviewPublicationBatchStore { ), ); return { + items: rows.map((row) => ({ + itemKey: String(row.item_key), + batchId: String(row.batch_id), + })), itemKeys: rows.map((row) => String(row.item_key)), activeBatches: new Set(rows.map((row) => String(row.batch_id))).size, nextLeaseExpiresAt: rows.length diff --git a/dashboard/exact-review-queue.ts b/dashboard/exact-review-queue.ts index d54a5370bf..46c0ef74f3 100644 --- a/dashboard/exact-review-queue.ts +++ b/dashboard/exact-review-queue.ts @@ -1880,7 +1880,10 @@ export class ExactReviewQueue { stateWriterCoordinatorQueuedStaleMs(this.env), ); const publicationBatches = this.batchStore.stats(now); - const batchOwnedItemKeys = new Set(publicationBatches.activeItemKeys); + const batchByItemKey = new Map( + publicationBatches.activeItemBatches.map((batch) => [batch.itemKey, batch] as const), + ); + const batchOwnedItemKeys = new Set(batchByItemKey.keys()); const freshPublicationItemKeys = this.freshPublicationItemKeysSync(state, now); const legacyExcludedItemKeys = new Set(batchOwnedItemKeys); if (exactReviewPublicationBatchingEnabled(this.env)) { @@ -1923,7 +1926,7 @@ export class ExactReviewQueue { bay_projection: exactReviewQueueBayProjection( Object.values(state.items), bayPriorityKeys, - batchOwnedItemKeys, + batchByItemKey, ), lanes: { review: { @@ -7287,11 +7290,17 @@ type ExactReviewBayProjectionItem = { created_at: string; updated_at: string; next_attempt_at: string; + batch_id?: string; + batch_created_at?: string; +}; + +type ExactReviewBayBatchOwner = { + batchId: string; }; function exactReviewQueueBayStage( item: ExactReviewQueueItem, - batchOwnedItemKeys: ReadonlySet = new Set(), + batchByItemKey: ReadonlyMap = new Map(), ): ExactReviewBayStage { // A parked item is deliberately no longer making normal queue progress. This // includes bounded review-retry exhaustion, permanent dispatch rejection, @@ -7301,7 +7310,7 @@ function exactReviewQueueBayStage( // The batch publisher's GitHub job is intentionally targetless. Its durable // batch membership is the authoritative bounded source for the individual // items it is currently applying, without another GitHub lookup. - if (batchOwnedItemKeys.has(item.key)) return "applying"; + if (batchByItemKey.has(item.key)) return "applying"; if (exactReviewQueueIsPublication(item)) return "publishing"; if (isLowPriorityExactReviewDecision(item.decision)) return "repairing"; return item.state === "pending" ? "arriving" : "setting-up"; @@ -7325,7 +7334,7 @@ function exactReviewQueueBayPriorityKeys(values: string[]) { function exactReviewQueueBayProjection( items: ExactReviewQueueItem[], priorityItemKeys: string[] = [], - batchOwnedItemKeys: ReadonlySet = new Set(), + batchByItemKey: ReadonlyMap = new Map(), ) { const projected = new Map(); for (const item of items) { @@ -7335,15 +7344,21 @@ function exactReviewQueueBayProjection( const repository = String(item.decision.targetRepo || "").trim(); const itemNumber = Number(item.decision.itemNumber); if (!repository || !Number.isSafeInteger(itemNumber) || itemNumber <= 0) continue; + const batch = batchByItemKey.get(item.key); const candidate: ExactReviewBayProjectionItem = { item_key: `${repository}#${itemNumber}`, repository, item_number: itemNumber, - stage: exactReviewQueueBayStage(item, batchOwnedItemKeys), + stage: exactReviewQueueBayStage(item, batchByItemKey), queue_state: item.state, created_at: new Date(item.createdAt).toISOString(), updated_at: new Date(item.updatedAt).toISOString(), next_attempt_at: new Date(item.nextAttemptAt).toISOString(), + ...(batch + ? { + batch_id: batch.batchId, + } + : {}), }; const previous = projected.get(candidate.item_key); const candidateUpdatedAt = Date.parse(candidate.updated_at); diff --git a/docs/proof/openclaw-bay/run-proof.mjs b/docs/proof/openclaw-bay/run-proof.mjs index bdb6c7307b..b0fb07e3a5 100644 --- a/docs/proof/openclaw-bay/run-proof.mjs +++ b/docs/proof/openclaw-bay/run-proof.mjs @@ -446,7 +446,12 @@ const batchApplyingProjection = queueProjection(); const batchApplyingItemKey = "openclaw/openclaw#108003"; const batchApplyingItems = batchApplyingProjection.bay_projection.items.map((item) => item.item_key === batchApplyingItemKey - ? { ...item, stage: "applying", queue_state: "pending" } + ? { + ...item, + stage: "applying", + queue_state: "pending", + batch_id: "exact-review-batch:9001", + } : item, ); proofSnapshots.push({ @@ -1032,9 +1037,9 @@ try { await currentStateWriterFailure; await page.waitForFunction(() => /history gap · awaiting current sample/i.test( - [...document.querySelectorAll("#bay-control-board .bay-control-card")] - .find((card) => /State writer/i.test(card.textContent || "")) - ?.textContent || "", + [...document.querySelectorAll("#bay-control-board .bay-control-card")].find((card) => + /State writer/i.test(card.textContent || ""), + )?.textContent || "", ), ); const failedStateWriterCopy = await page @@ -1732,12 +1737,19 @@ try { await page.evaluate(async () => { await window.__bayProofPoll(); }); - const batchPublisherLane = page.locator(`[data-stage="applying"] [data-key="${batchApplyingItemKey}"]`); + const batchPublisherLane = page.locator( + `[data-stage="applying"] [data-key="${batchApplyingItemKey}"]`, + ); await batchPublisherLane.waitFor({ state: "visible", timeout: 5_000 }); assertProof( - "durable batch ownership places known publication items in Applying", - (await batchPublisherLane.count()) === 1, - { item_key: batchApplyingItemKey, stage: "applying" }, + "durable batch ownership places known publication items in Applying & writing", + (await batchPublisherLane.count()) === 1 && + /APPLYING & WRITING 1/.test(await page.locator('[data-stage="applying"] h2').innerText()), + { + item_key: batchApplyingItemKey, + stage: "applying", + batch_id: "exact-review-batch:9001", + }, ); fixtureIndex = 10; diff --git a/test/dashboard-worker.test.ts b/test/dashboard-worker.test.ts index feadf30a83..417038c0d2 100644 --- a/test/dashboard-worker.test.ts +++ b/test/dashboard-worker.test.ts @@ -2147,8 +2147,15 @@ test("Bay queue projection maps durable batch-owned publication items to Applyin status.bay_projection.items.map((item) => ({ item_key: item.item_key, stage: item.stage, + batch_id: item.batch_id, })), - [{ item_key: "openclaw/gogcli#602", stage: "applying" }], + [ + { + item_key: "openclaw/gogcli#602", + stage: "applying", + batch_id: "bay-applying-batch", + }, + ], ); }); diff --git a/test/exact-review-publication-batches.test.ts b/test/exact-review-publication-batches.test.ts index 686f226afc..ed88235cb6 100644 --- a/test/exact-review-publication-batches.test.ts +++ b/test/exact-review-publication-batches.test.ts @@ -133,6 +133,10 @@ test("publication batches atomically select ready items without duplicate active assert.equal(second, null); assert.equal(first?.configuredBatchSize, 2); assert.deepEqual(batches.activeLeaseSnapshot(1_500), { + items: candidates.slice(0, 2).map((candidate) => ({ + itemKey: candidate.itemKey, + batchId: "batch-1", + })), itemKeys: candidates.slice(0, 2).map((item) => item.itemKey), activeBatches: 1, nextLeaseExpiresAt: 2_000, @@ -183,6 +187,16 @@ test("publication batches allow a bounded number of disjoint active owners", () ); assert.equal(third, null); assert.deepEqual(batches.activeLeaseSnapshot(1_500), { + items: [ + ...candidates.slice(0, 2).map((candidate) => ({ + itemKey: candidate.itemKey, + batchId: "parallel-1", + })), + ...candidates.slice(2).map((candidate) => ({ + itemKey: candidate.itemKey, + batchId: "parallel-2", + })), + ], itemKeys: candidates.map((item) => item.itemKey), activeBatches: 2, nextLeaseExpiresAt: 2_000,