Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 31 additions & 10 deletions dashboard/bay-page.ts

Large diffs are not rendered by default.

75 changes: 75 additions & 0 deletions dashboard/exact-review-health.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,25 @@ export type ExactReviewHandoffHealth = {
phases: Record<ExactReviewPhase, ExactReviewPhaseSummary>;
};

export type ExactReviewPressureStatus = "idle" | "congested" | "saturated" | "unknown";

export type ExactReviewPressureSummary = {
status: ExactReviewPressureStatus;
reason:
| "capacity_unavailable"
| "capacity_available"
| "no_ready_backlog"
| "no_admissible_backlog"
| "dispatcher_inactive"
| "handoff_unknown"
| "capacity_full_with_backlog";
capacity: number;
active: number;
pending: number;
ready_pending: number;
admissible_pending: number;
};

const PHASES: ExactReviewPhase[] = ["pending", "dispatching", "leased"];

export function summarizeExactReviewHandoff({
Expand Down Expand Up @@ -176,6 +195,57 @@ export function summarizeExactReviewHandoff({
};
}

export function summarizeExactReviewPressure({
pending,
readyPending,
admissiblePending,
dispatching,
leased,
capacity,
dispatcherState,
handoffStatus,
}: {
pending: number;
readyPending: number;
admissiblePending: number;
dispatching: number;
leased: number;
capacity: number;
dispatcherState?: string;
handoffStatus?: string;
}): ExactReviewPressureSummary {
const safePending = nonNegativeInteger(pending);
const safeReadyPending = Math.min(safePending, nonNegativeInteger(readyPending));
const safeAdmissiblePending = Math.min(safeReadyPending, nonNegativeInteger(admissiblePending));
const safeCapacity = nonNegativeInteger(capacity);
const active = nonNegativeInteger(dispatching) + nonNegativeInteger(leased);
const common = {
capacity: safeCapacity,
active,
pending: safePending,
ready_pending: safeReadyPending,
admissible_pending: safeAdmissiblePending,
};

if (safeCapacity < 1) return { status: "unknown", reason: "capacity_unavailable", ...common };
if (safeReadyPending < 1) return { status: "idle", reason: "no_ready_backlog", ...common };
if (safeAdmissiblePending < 1) {
return { status: "idle", reason: "no_admissible_backlog", ...common };
}
if (active < safeCapacity) return { status: "idle", reason: "capacity_available", ...common };
if (dispatcherState !== "active") {
return { status: "unknown", reason: "dispatcher_inactive", ...common };
}
if (!["healthy", "degraded", "stalled"].includes(String(handoffStatus || ""))) {
return { status: "unknown", reason: "handoff_unknown", ...common };
}
return {
status: safeAdmissiblePending >= safeCapacity ? "saturated" : "congested",
reason: "capacity_full_with_backlog",
...common,
};
}

function exactReviewPhaseStartedAt(
item: ExactReviewHealthItem,
now: number,
Expand Down Expand Up @@ -230,3 +300,8 @@ function finiteNumber(value: unknown, fallback: number) {
const number = Number(value);
return Number.isFinite(number) ? number : fallback;
}

function nonNegativeInteger(value: unknown) {
const number = Number(value);
return Number.isFinite(number) ? Math.max(0, Math.floor(number)) : 0;
}
138 changes: 137 additions & 1 deletion dashboard/exact-review-queue.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { stableJson } from "../src/stable-json.ts";
import { summarizeExactReviewHandoff } from "./exact-review-health.ts";
import {
summarizeExactReviewHandoff,
summarizeExactReviewPressure,
} from "./exact-review-health.ts";

type GithubAppJsonOptions = { method?: string; body?: BodyInit; errorLabel?: string };
const GITHUB_TIMEOUT_MS = 4500;
Expand Down Expand Up @@ -3401,6 +3404,110 @@ function exactReviewQueueLane(item: ExactReviewQueueItem) {
return exactReviewQueueIsPublication(item) ? "publication" : "review";
}

// The Bay is a deliberately lightweight visual projection of durable queue
// state. Keep this representation bounded and scrubbed: it is public dashboard
// data, not a queue-inspection API. Live workers remain the authority for the
// reviewing stage; these records only make the otherwise invisible admission,
// setup, publication, and recovery phases visible.
const EXACT_REVIEW_BAY_SAMPLE_LIMIT = 24;
const EXACT_REVIEW_BAY_STAGES = [
"arriving",
"setting-up",
"reviewing",
"applying",
"repairing",
] as const;
type ExactReviewBayStage = (typeof EXACT_REVIEW_BAY_STAGES)[number];
type ExactReviewBayProjectionItem = {
item_key: string;
repository: string;
item_number: number;
stage: ExactReviewBayStage;
queue_state: ExactReviewQueueItem["state"];
created_at: string;
updated_at: string;
next_attempt_at: string;
};

function exactReviewQueueBayStage(item: ExactReviewQueueItem): ExactReviewBayStage {
if (exactReviewQueueIsPublication(item)) return "applying";
if (isLowPriorityExactReviewDecision(item.decision)) return "repairing";
return item.state === "pending" ? "arriving" : "setting-up";
}

function exactReviewQueueBayStagePriority(stage: ExactReviewBayStage) {
return EXACT_REVIEW_BAY_STAGES.indexOf(stage);
}

function exactReviewQueueBayProjection(items: ExactReviewQueueItem[]) {
const projected = new Map<string, ExactReviewBayProjectionItem>();
for (const item of items) {
if (item.state === "parked") continue;
const repository = String(item.decision.targetRepo || "").trim();
const itemNumber = Number(item.decision.itemNumber);
if (!repository || !Number.isSafeInteger(itemNumber) || itemNumber <= 0) continue;
const candidate: ExactReviewBayProjectionItem = {
item_key: `${repository}#${itemNumber}`,
repository,
item_number: itemNumber,
stage: exactReviewQueueBayStage(item),
queue_state: item.state,
created_at: new Date(item.createdAt).toISOString(),
updated_at: new Date(item.updatedAt).toISOString(),
next_attempt_at: new Date(item.nextAttemptAt).toISOString(),
};
const previous = projected.get(candidate.item_key);
const candidateUpdatedAt = Date.parse(candidate.updated_at);
const previousUpdatedAt = previous ? Date.parse(previous.updated_at) : Number.NEGATIVE_INFINITY;
if (
!previous ||
candidateUpdatedAt > previousUpdatedAt ||
(candidateUpdatedAt === previousUpdatedAt &&
exactReviewQueueBayStagePriority(candidate.stage) >
exactReviewQueueBayStagePriority(previous.stage))
) {
projected.set(candidate.item_key, candidate);
}
}
const rows = [...projected.values()];
const stages = Object.fromEntries(
EXACT_REVIEW_BAY_STAGES.map((stage) => [
stage,
rows.filter((item) => item.stage === stage).length,
]),
) as Record<ExactReviewBayStage, number>;
const rowsByStage = Object.fromEntries(
EXACT_REVIEW_BAY_STAGES.map((stage) => [
stage,
rows
.filter((item) => item.stage === stage)
.sort(
(left, right) =>
Date.parse(left.created_at) - Date.parse(right.created_at) ||
left.item_key.localeCompare(right.item_key),
),
]),
) as Record<ExactReviewBayStage, ExactReviewBayProjectionItem[]>;
const sample: ExactReviewBayProjectionItem[] = [];
for (let index = 0; sample.length < EXACT_REVIEW_BAY_SAMPLE_LIMIT; index += 1) {
let added = false;
for (const stage of EXACT_REVIEW_BAY_STAGES) {
const item = rowsByStage[stage][index];
if (!item) continue;
sample.push(item);
added = true;
if (sample.length === EXACT_REVIEW_BAY_SAMPLE_LIMIT) break;
}
if (!added) break;
}
return {
sample_limit: EXACT_REVIEW_BAY_SAMPLE_LIMIT,
total: rows.length,
stages,
items: sample,
};
}

function exactReviewQueueActiveReviewCount(state: ExactReviewQueueState) {
return Object.values(state.items).filter(
(item) =>
Expand Down Expand Up @@ -3559,8 +3666,35 @@ function exactReviewQueueStats(
publicationCapacity,
),
};
const readyPending = items.filter(
(item) => item.state === "pending" && item.nextAttemptAt <= now,
).length;
const admissibleItems = exactReviewQueueAdmittedItems(
state,
now,
Number.MAX_SAFE_INTEGER,
targetCapacity,
publicationCapacity,
);
const admissiblePending = admissibleItems.length;
const reviewAdmissiblePending = admissibleItems.filter(
(item) => !exactReviewQueueIsPublication(item),
).length;
const pressure = summarizeExactReviewPressure({
pending: lanes.review.pending,
readyPending: lanes.review.ready,
admissiblePending: reviewAdmissiblePending,
dispatching: lanes.review.dispatching,
leased: lanes.review.leased,
capacity: lanes.review.capacity,
dispatcherState: state.dispatcher?.state,
handoffStatus: handoffHealth.status,
});
return {
generated_at: handoffHealth.observed_at,
pending: handoffHealth.phases.pending.count,
ready_pending: readyPending,
admissible_pending: admissiblePending,
shed_since_reset: exactReviewShedSinceReset(state),
dispatching: handoffHealth.phases.dispatching.count,
leased: handoffHealth.phases.leased.count,
Expand All @@ -3573,6 +3707,8 @@ function exactReviewQueueStats(
oldest_leased_age_seconds: handoffHealth.phases.leased.oldest_age_seconds,
handoff_health: handoffHealth,
lanes,
pressure,
bay_projection: exactReviewQueueBayProjection(items),
next_wake_at: nextWakeAt === null ? null : new Date(nextWakeAt).toISOString(),
dispatcher: {
state: state.dispatcher?.state || "unknown",
Expand Down
14 changes: 11 additions & 3 deletions dashboard/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7360,8 +7360,11 @@ h2::before { content: ""; flex: 0 0 auto; width: 14px; height: 2px; border-radiu
}
.health-badge.healthy,
.health-badge.idle { color: var(--green); border-color: color-mix(in srgb, var(--green) 40%, transparent); }
.health-badge.degraded { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 45%, transparent); }
.health-badge.stalled { color: var(--red); border-color: color-mix(in srgb, var(--red) 45%, transparent); }
.health-badge.degraded,
.health-badge.congested { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 45%, transparent); }
.health-badge.stalled,
.health-badge.saturated { color: var(--red); border-color: color-mix(in srgb, var(--red) 45%, transparent); }
.exact-handoff-badges { display: flex; flex: 0 0 auto; flex-wrap: wrap; gap: 6px; justify-content: end; }
.handoff-phases {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
Expand Down Expand Up @@ -8652,6 +8655,10 @@ function renderExactReviewHandoff(queue) {
return;
}
const status = ["idle", "healthy", "degraded", "stalled"].includes(health.status) ? health.status : "unknown";
const pressure = queue?.pressure;
const pressureStatus = ["idle", "congested", "saturated", "unknown"].includes(pressure?.status)
? pressure.status
: "unknown";
const labels = {
pending: ["Pending", "waiting for admission"],
dispatching: ["Dispatching", "waiting for run claim"],
Expand All @@ -8665,8 +8672,9 @@ function renderExactReviewHandoff(queue) {
return '<div class="handoff-phase"><span>' + esc(labels[phase][0]) + '</span><strong>' + fmt.format(summary.count || 0) + '</strong><small>' + esc(labels[phase][1] + " · " + age) + '</small></div>';
}).join("");
const slots = fmt.format(health.available_slots || 0) + " of " + fmt.format(health.capacity || 0) + " exact-review slots open";
const backlog = fmt.format(queue?.pending || 0) + " total · " + fmt.format(queue?.ready_pending || 0) + " ready · " + fmt.format(queue?.admissible_pending || 0) + " admissible";
const threshold = "stalled after " + elapsed((health.stalled_after_seconds || 0) * 1000);
target.innerHTML = '<div class="exact-handoff"><div class="exact-handoff-head"><div class="exact-handoff-title"><strong>Queue handoff health</strong><span>' + esc(health.message || "Queue phase telemetry") + '</span></div><span class="health-badge ' + esc(status) + '">' + esc(status) + '</span></div><div class="handoff-phases">' + phases + '</div><div class="handoff-foot"><span>' + esc(slots) + '</span><span>' + esc(threshold) + '</span></div></div>';
target.innerHTML = '<div class="exact-handoff"><div class="exact-handoff-head"><div class="exact-handoff-title"><strong>Queue handoff health</strong><span>' + esc(health.message || "Queue phase telemetry") + '</span></div><div class="exact-handoff-badges"><span class="health-badge ' + esc(status) + '">' + esc(status) + '</span><span class="health-badge ' + esc(pressureStatus) + '">pressure ' + esc(pressureStatus) + '</span></div></div><div class="handoff-phases">' + phases + '</div><div class="handoff-foot"><span>' + esc(slots) + '</span><span>' + esc(backlog) + '</span><span>' + esc(threshold) + '</span></div></div>';
}
function renderWorkers(rows) {
workerIndex = new Map(rows.map(worker => [String(worker.id), worker]));
Expand Down
16 changes: 13 additions & 3 deletions docs/live-dashboard.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ is absent or a cache event lands in another Cloudflare colo.
apply-ready candidate count and an estimated number of cursor windows to
revisit the close queue; scheduled cadence time is explanatory only because
successful windows can dispatch immediate continuations
- exact-review queue backlog, retry-ready backlog, target-admissible backlog,
and pressure classification from the current durable queue snapshot

The Worker fetches job details only for the bounded active-run set, limits that
GitHub fanout to 12 concurrent requests, and caches each run's jobs for 60
Expand Down Expand Up @@ -287,9 +289,17 @@ workflow state, check time, and retry time so an intentional pause cannot look
like occupied executor capacity. Re-enabling the workflow does not require a
queue mutation; the next status check resumes normal admission.

The same endpoint exposes `handoff_health` plus oldest timestamps and ages for
the pending, dispatching, and leased phases. New dispatch and claim transitions
carry explicit phase timestamps. Rows written by an older deployment derive
The same endpoint exposes `generated_at`, `ready_pending`,
`admissible_pending`, `pressure`, `handoff_health`, and oldest timestamps and
ages for the pending, dispatching, and leased phases. `ready_pending` excludes
retry-delayed items. `admissible_pending` further excludes ready items blocked
by their target's exact-review cap. `pressure` is a deterministic observation
from that same queue snapshot: it reports `congested` or `saturated` only when
capacity is full, the dispatcher and handoff telemetry are known, and
target-admissible backlog remains. The snapshot adds no GitHub API fanout, and
no workflow, planner, admission, continuation, or dispatch decision consumes
the pressure value. New dispatch and claim transitions carry explicit phase
timestamps. Rows written by an older deployment derive
their phase start from the active dispatch or execution lease; a stale timestamp
left by a rollback cannot override that newer lease, and a wholly unknown legacy
age stays non-alarming. A claim is degraded after one third of the dispatch
Expand Down
Loading