From 84f66037b39fc5f67a143951c5f2c5b743aa8d06 Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 16:54:27 +0200 Subject: [PATCH 01/10] feat(discordbot): answer bare "status" mentions from the control plane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A "@bot status" (or "health") mention now gets an instant reply built from api-rs /healthz + /readyz and the shared session database (recent turns with errors, 24h tally, in-flight executions, active sandboxes, warm pool) — no sandbox turn involved, so the command keeps working when the agent pipeline itself is what's broken. Mentions with any other words ("status of the deploy") fall through to a normal agent turn, so real questions are never hijacked. Every status source is fetched independently and best-effort: api-rs down still reports DB data and vice versa. Co-Authored-By: Claude Fable 5 --- services/discordbot/src/index.ts | 52 +++++ services/discordbot/src/status.ts | 295 ++++++++++++++++++++++++ services/discordbot/test/status.test.ts | 217 +++++++++++++++++ 3 files changed, 564 insertions(+) create mode 100644 services/discordbot/src/status.ts create mode 100644 services/discordbot/test/status.test.ts diff --git a/services/discordbot/src/index.ts b/services/discordbot/src/index.ts index 4491405c8..6b61ddb4b 100644 --- a/services/discordbot/src/index.ts +++ b/services/discordbot/src/index.ts @@ -33,6 +33,7 @@ import { renameThreadFromMessage, } from "./discord-threading"; import { setGatewayConnected } from "./gateway"; +import { collectStatus, formatStatus, isStatusCommand } from "./status"; import { collectInitialContext, executeSessionTurn, @@ -259,8 +260,57 @@ export function createDiscordbot(options: DiscordbotOptions): Discordbot { DEFAULT_MAX_CONCURRENT_EXECUTIONS_PER_GUILD, ); + // Lazy pool for the status fast-path, separate from the state adapter's pool + // (which is private to createDefaultState — and options.state deployments + // have no pool at all). Capped small: status is an occasional human command. + let statusPool: pg.Pool | null = null; + const statusDb = (): pg.Pool | null => { + if (!options.postgresUrl) return null; + if (!statusPool) { + statusPool = new pg.Pool({ + connectionString: options.postgresUrl, + max: 2, + }); + statusPool.on("error", (error) => { + logger.warn("discordbot_status_pool_error", { + error: errorMessage(error), + }); + }); + } + return statusPool; + }; + + // A bare "status"/"health" mention answers straight from the control plane + // (api-rs health + the shared session DB) with no sandbox turn — so it still + // works when the agent pipeline is what's broken. Returns true when handled. + const maybeReplyStatus = async ( + thread: Thread, + message: ChatMessage, + ): Promise => { + if (!isStatusCommand(message.text ?? "")) return false; + try { + const report = await collectStatus({ + apiUrl: options.apiUrl, + db: statusDb(), + fetchFn: options.fetch, + }); + await thread.post(formatStatus(report)); + } catch (error) { + logger.warn("discordbot_status_reply_failed", { + error: errorMessage(error), + }); + try { + await thread.post(`⚠️ status check failed: ${errorMessage(error)}`); + } catch { + // best-effort; nothing left to signal with. + } + } + return true; + }; + chat.onNewMention(async (thread, message) => { if (!isAllowedDiscordMessage(message, options, logger)) return; + if (await maybeReplyStatus(thread, message)) return; await thread.subscribe(); await syncThreadMessageToSession(thread, message, { executionLimiter, @@ -272,6 +322,8 @@ export function createDiscordbot(options: DiscordbotOptions): Discordbot { chat.onSubscribedMessage(async (thread, message) => { if (!isAllowedDiscordMessage(message, options, logger)) return; + if (message.isMention === true && (await maybeReplyStatus(thread, message))) + return; await syncThreadMessageToSession(thread, message, { executionLimiter, mode: message.isMention === true ? "execute" : "append", diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts new file mode 100644 index 000000000..607cd43f6 --- /dev/null +++ b/services/discordbot/src/status.ts @@ -0,0 +1,295 @@ +import type { DiscordbotFetch } from "./types"; +import { errorMessage, sliceSurrogateSafe } from "./utils"; + +// A "status" mention answers directly from the control plane — no sandbox, no +// session turn. The whole point is that it still works when the agent pipeline +// is broken: api-rs health comes from its /healthz + /readyz endpoints, and the +// turn/sandbox history comes from the shared session database (the same +// Postgres api-rs writes session_executions/sessions/session_warm_sandboxes +// to, reached via the bot's DATABASE_URL). Every source is fetched +// independently and best-effort so one dead dependency never blanks the rest +// of the report. + +/** Structural slice of pg.Pool so tests can stub the database trivially. */ +export type StatusDb = { + query(sql: string): Promise<{ rows: Record[] }>; +}; + +const KEYWORD = /^(status|health)[?!.]*$/i; +// Raw Discord mention markup (<@123>, <@!123>, <@&role>, <#channel>) plus the +// adapter's rewritten form (`@name`): none of it counts as words. +const MENTION_TOKEN = /^(<[@#][!&]?\w+>|@[\w.-]+)$/; + +/** + * True when the message is ONLY a status request ("@gerard status", + * "<@&123> health?"). Anything with more words ("status of the deploy") falls + * through to a normal agent turn so real questions are never hijacked. + */ +export function isStatusCommand(text: string): boolean { + const words = text + .split(/\s+/) + .filter((word) => word.length > 0 && !MENTION_TOKEN.test(word)); + return words.length === 1 && KEYWORD.test(words[0] ?? ""); +} + +export type ExecutionRow = { + ageSeconds: number | null; + durationSeconds: number | null; + error: string; + status: string; + threadKey: string; +}; + +export type StatusReport = { + apiHealthy: boolean | null; + apiReady: boolean | null; + collectedNotes: string[]; + dbOk: boolean; + inFlight: ExecutionRow[]; + recent: ExecutionRow[]; + sandboxes: { ageSeconds: number | null; sandboxId: string; threadKey: string }[]; + tally: Record; + warmPool: Record; +}; + +const HEALTH_TIMEOUT_MS = 2_000; +const ERROR_SNIPPET_CHARS = 150; + +export async function collectStatus(input: { + apiUrl: string; + db: StatusDb | null; + fetchFn?: DiscordbotFetch; + nowMs?: number; +}): Promise { + const fetchFn = input.fetchFn ?? fetch; + const now = input.nowMs ?? Date.now(); + const notes: string[] = []; + + const probe = async (path: string): Promise => { + try { + const response = await fetchFn(`${input.apiUrl}${path}`, { + signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS), + }); + return response.ok; + } catch { + return false; + } + }; + + const query = async ( + label: string, + sql: string, + ): Promise[] | null> => { + if (!input.db) return null; + try { + return (await input.db.query(sql)).rows; + } catch (error) { + notes.push(`${label} unavailable (${errorMessage(error)})`); + return null; + } + }; + + const toExecutionRow = (row: Record): ExecutionRow => ({ + ageSeconds: ageSecondsFrom(row.created_at, now), + durationSeconds: numberOrNull(row.duration_seconds), + error: String(row.error ?? "").slice(0, ERROR_SNIPPET_CHARS), + status: String(row.status ?? "unknown"), + threadKey: String(row.thread_key ?? "?"), + }); + + const [apiHealthy, apiReady, recent, tally, inFlight, sandboxes, warm] = + await Promise.all([ + probe("/healthz"), + probe("/readyz"), + query( + "recent turns", + `SELECT thread_key, status, left(coalesce(error, ''), ${ERROR_SNIPPET_CHARS}) AS error, + created_at, + extract(epoch FROM (completed_at - started_at)) AS duration_seconds + FROM session_executions + ORDER BY created_at DESC + LIMIT 8`, + ), + query( + "24h tally", + `SELECT status, count(*)::int AS count + FROM session_executions + WHERE created_at > now() - interval '24 hours' + GROUP BY status`, + ), + query( + "in-flight turns", + `SELECT thread_key, status, '' AS error, created_at, + NULL AS duration_seconds + FROM session_executions + WHERE status IN ('queued', 'running') + ORDER BY created_at ASC + LIMIT 8`, + ), + query( + "active sandboxes", + `SELECT thread_key, sandbox_id, sandbox_last_active_at + FROM sessions + WHERE sandbox_id IS NOT NULL + AND sandbox_last_active_at > now() - interval '2 hours' + ORDER BY sandbox_last_active_at DESC + LIMIT 8`, + ), + query( + "warm pool", + `SELECT status, count(*)::int AS count + FROM session_warm_sandboxes + GROUP BY status`, + ), + ]); + + return { + apiHealthy, + apiReady, + collectedNotes: notes, + dbOk: recent !== null, + inFlight: (inFlight ?? []).map(toExecutionRow), + recent: (recent ?? []).map(toExecutionRow), + sandboxes: (sandboxes ?? []).map((row) => ({ + ageSeconds: ageSecondsFrom(row.sandbox_last_active_at, now), + sandboxId: String(row.sandbox_id ?? "?"), + threadKey: String(row.thread_key ?? "?"), + })), + tally: countsByStatus(tally), + warmPool: countsByStatus(warm), + }; +} + +// Discord caps messages at 2000 chars; stay under it with honest truncation. +const STATUS_MAX_CHARS = 1_900; + +const STATUS_EMOJI: Record = { + cancelled: "🚫", + completed: "✅", + failed: "❌", + queued: "🕒", + running: "▶️", +}; + +export function formatStatus(report: StatusReport): string { + const lines: string[] = []; + + const mark = (value: boolean | null): string => + value === null ? "❓" : value ? "✅" : "❌"; + lines.push( + `**gerard status** · api-rs ${mark(report.apiHealthy)} ` + + `ready ${mark(report.apiReady)} · db ${report.dbOk ? "✅" : "❌"}`, + ); + + const tallyEntries = Object.entries(report.tally).sort(); + if (tallyEntries.length > 0) { + lines.push( + `last 24h: ${tallyEntries + .map(([status, count]) => `${count} ${STATUS_EMOJI[status] ?? status}`) + .join(" · ")}`, + ); + } + + if (report.inFlight.length > 0) { + lines.push("in flight:"); + for (const row of report.inFlight) { + lines.push( + `${STATUS_EMOJI[row.status] ?? "•"} ${describeThread(row.threadKey)}` + + ` (${formatAge(row.ageSeconds)})`, + ); + } + } + + if (report.recent.length > 0) { + lines.push("recent turns:"); + for (const row of report.recent) { + const duration = + row.durationSeconds !== null + ? ` (${formatDuration(row.durationSeconds)})` + : ""; + const error = row.error ? ` — ${row.error}` : ""; + lines.push( + `${STATUS_EMOJI[row.status] ?? "•"} ${formatAge(row.ageSeconds)} ago · ` + + `${describeThread(row.threadKey)}${duration}${error}`, + ); + } + } + + const warmEntries = Object.entries(report.warmPool).sort(); + const sandboxBits: string[] = []; + if (report.sandboxes.length > 0) { + sandboxBits.push(`${report.sandboxes.length} active`); + } + if (warmEntries.length > 0) { + sandboxBits.push( + `warm: ${warmEntries + .map(([status, count]) => `${count} ${status}`) + .join(", ")}`, + ); + } + if (sandboxBits.length > 0) lines.push(`sandboxes: ${sandboxBits.join(" · ")}`); + + for (const note of report.collectedNotes) lines.push(`⚠️ ${note}`); + if (!report.dbOk && report.collectedNotes.length === 0) { + lines.push("⚠️ session database unreachable — turn history unavailable"); + } + + const text = lines.join("\n"); + if (text.length <= STATUS_MAX_CHARS) return text; + return `${sliceSurrogateSafe(text, STATUS_MAX_CHARS - 12).trimEnd()}\n[truncated]`; +} + +/** `platform · short thread name` from a session thread key. */ +function describeThread(threadKey: string): string { + const separator = threadKey.indexOf(":"); + if (separator === -1) return shorten(threadKey); + const platform = threadKey.slice(0, separator); + const rest = threadKey.slice(separator + 1); + return `${platform} ${shorten(rest)}`; +} + +function shorten(value: string): string { + return value.length <= 40 ? value : `…${value.slice(-39)}`; +} + +function formatAge(seconds: number | null): string { + if (seconds === null) return "?"; + return formatDuration(seconds); +} + +function formatDuration(seconds: number): string { + const s = Math.max(0, Math.round(seconds)); + if (s < 60) return `${s}s`; + if (s < 3600) return `${Math.round(s / 60)}m`; + if (s < 86400) return `${Math.round(s / 3600)}h`; + return `${Math.round(s / 86400)}d`; +} + +function countsByStatus( + rows: Record[] | null, +): Record { + const counts: Record = {}; + for (const row of rows ?? []) { + const count = numberOrNull(row.count); + if (count !== null) counts[String(row.status ?? "unknown")] = count; + } + return counts; +} + +function ageSecondsFrom(value: unknown, nowMs: number): number | null { + if (value instanceof Date) return (nowMs - value.getTime()) / 1000; + if (typeof value === "string") { + const parsed = Date.parse(value); + if (!Number.isNaN(parsed)) return (nowMs - parsed) / 1000; + } + return null; +} + +function numberOrNull(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim() !== "") { + const parsed = Number(value); + if (Number.isFinite(parsed)) return parsed; + } + return null; +} diff --git a/services/discordbot/test/status.test.ts b/services/discordbot/test/status.test.ts new file mode 100644 index 000000000..dc01b2c5d --- /dev/null +++ b/services/discordbot/test/status.test.ts @@ -0,0 +1,217 @@ +import { describe, expect, it } from "bun:test"; +import { + collectStatus, + formatStatus, + isStatusCommand, + type StatusDb, + type StatusReport, +} from "../src/status"; +import type { DiscordbotFetch } from "../src/types"; + +describe("isStatusCommand", () => { + it("matches bare status/health requests with mention markup", () => { + expect(isStatusCommand("status")).toBe(true); + expect(isStatusCommand("Status?")).toBe(true); + expect(isStatusCommand("health!")).toBe(true); + expect(isStatusCommand("<@123456> status")).toBe(true); + expect(isStatusCommand("<@!123456> health")).toBe(true); + expect(isStatusCommand("<@&987> status")).toBe(true); + expect(isStatusCommand("@gerard status")).toBe(true); + expect(isStatusCommand(" @gerard STATUS ")).toBe(true); + }); + + it("rejects real questions and ordinary messages", () => { + expect(isStatusCommand("status of the deploy")).toBe(false); + expect(isStatusCommand("@gerard what's the status?")).toBe(false); + expect(isStatusCommand("can you check the health of api-rs")).toBe(false); + expect(isStatusCommand("hello")).toBe(false); + expect(isStatusCommand("")).toBe(false); + expect(isStatusCommand("<@123456>")).toBe(false); + }); +}); + +function healthyFetch(status = 200): DiscordbotFetch { + return async () => new Response("ok", { status }); +} + +function stubDb(handler: (sql: string) => Record[]): StatusDb { + return { + async query(sql: string) { + return { rows: handler(sql) }; + }, + }; +} + +const NOW = Date.parse("2026-08-12T12:00:00Z"); + +function fullDb(): StatusDb { + return stubDb((sql) => { + if (sql.includes("interval '24 hours'")) { + return [ + { status: "completed", count: 41 }, + { status: "failed", count: 2 }, + ]; + } + if (sql.includes("IN ('queued', 'running')")) { + return [ + { + created_at: new Date(NOW - 120_000), + duration_seconds: null, + error: "", + status: "running", + thread_key: "discord:1:2:3", + }, + ]; + } + if (sql.includes("FROM session_executions")) { + return [ + { + created_at: new Date(NOW - 300_000), + duration_seconds: "63", + error: "", + status: "completed", + thread_key: "github-manage:0xSplits/splits-teams:1799", + }, + { + created_at: new Date(NOW - 1_900_000), + duration_seconds: "12", + error: "sandbox spawn timeout after 120s", + status: "failed", + thread_key: "discord:1:2:9", + }, + ]; + } + if (sql.includes("FROM sessions")) { + return [ + { + sandbox_id: "asbx-1755000000-1", + sandbox_last_active_at: new Date(NOW - 60_000), + thread_key: "discord:1:2:3", + }, + ]; + } + if (sql.includes("session_warm_sandboxes")) { + return [{ status: "ready", count: 2 }]; + } + return []; + }); +} + +describe("collectStatus", () => { + it("assembles a full report when everything is up", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + db: fullDb(), + fetchFn: healthyFetch(), + nowMs: NOW, + }); + expect(report.apiHealthy).toBe(true); + expect(report.apiReady).toBe(true); + expect(report.dbOk).toBe(true); + expect(report.tally).toEqual({ completed: 41, failed: 2 }); + expect(report.recent).toHaveLength(2); + expect(report.recent[1]?.error).toContain("sandbox spawn timeout"); + expect(report.inFlight).toHaveLength(1); + expect(report.sandboxes[0]?.sandboxId).toBe("asbx-1755000000-1"); + expect(report.warmPool).toEqual({ ready: 2 }); + expect(report.collectedNotes).toEqual([]); + }); + + it("still reports DB data when api-rs is down", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + db: fullDb(), + fetchFn: async () => { + throw new Error("connect ECONNREFUSED"); + }, + nowMs: NOW, + }); + expect(report.apiHealthy).toBe(false); + expect(report.apiReady).toBe(false); + expect(report.dbOk).toBe(true); + expect(report.recent).toHaveLength(2); + }); + + it("still reports api-rs health when the DB is down", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + db: stubDb(() => { + throw new Error("password authentication failed"); + }), + fetchFn: healthyFetch(), + nowMs: NOW, + }); + expect(report.apiHealthy).toBe(true); + expect(report.dbOk).toBe(false); + expect(report.collectedNotes.length).toBeGreaterThan(0); + expect(report.recent).toEqual([]); + }); + + it("handles a missing database configuration", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + db: null, + fetchFn: healthyFetch(), + nowMs: NOW, + }); + expect(report.dbOk).toBe(false); + expect(report.recent).toEqual([]); + }); +}); + +describe("formatStatus", () => { + const baseReport = (): StatusReport => ({ + apiHealthy: true, + apiReady: true, + collectedNotes: [], + dbOk: true, + inFlight: [], + recent: [], + sandboxes: [], + tally: {}, + warmPool: {}, + }); + + it("renders health, tallies, turns, and sandboxes compactly", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + db: fullDb(), + fetchFn: healthyFetch(), + nowMs: NOW, + }); + const text = formatStatus(report); + expect(text).toContain("api-rs ✅"); + expect(text).toContain("41 ✅"); + expect(text).toContain("2 ❌"); + expect(text).toContain("github-manage 0xSplits/splits-teams:1799"); + expect(text).toContain("(1m)"); + expect(text).toContain("sandbox spawn timeout"); + expect(text).toContain("warm: 2 ready"); + expect(text.length).toBeLessThanOrEqual(2000); + }); + + it("marks a down api-rs and unreachable DB honestly", () => { + const report = baseReport(); + report.apiHealthy = false; + report.apiReady = false; + report.dbOk = false; + const text = formatStatus(report); + expect(text).toContain("api-rs ❌"); + expect(text).toContain("db ❌"); + expect(text).toContain("session database unreachable"); + }); + + it("stays under the Discord cap with oversized errors", () => { + const report = baseReport(); + report.recent = Array.from({ length: 12 }, (_, index) => ({ + ageSeconds: 60 * index, + durationSeconds: 5, + error: "x".repeat(150), + status: "failed", + threadKey: `discord:${"y".repeat(80)}:${index}`, + })); + const text = formatStatus(report); + expect(text.length).toBeLessThanOrEqual(2000); + expect(text.endsWith("[truncated]")).toBe(true); + }); +}); From f04c74c9ec1b74394a98cab5a39adf88e04b1e3f Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 18:03:46 +0200 Subject: [PATCH 02/10] feat(discordbot): tabular status reply in a code block Discord has no table markup; a monospace code block with padded columns is the idiomatic substitute. Header (bold + emoji health marks) stays outside the block, rows use short ASCII tags (ok/FAIL/run) since emoji are double-width in code blocks and wreck alignment. In-flight turns fold into the same table as settled ones; errors get their own indented line so the columns stay ~45 chars and portrait mobile doesn't wrap. Co-Authored-By: Claude Fable 5 --- services/discordbot/src/status.ts | 113 ++++++++++++++---------- services/discordbot/test/status.test.ts | 47 +++++++--- 2 files changed, 104 insertions(+), 56 deletions(-) diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts index 607cd43f6..33d07969d 100644 --- a/services/discordbot/src/status.ts +++ b/services/discordbot/src/status.ts @@ -163,55 +163,65 @@ export async function collectStatus(input: { // Discord caps messages at 2000 chars; stay under it with honest truncation. const STATUS_MAX_CHARS = 1_900; -const STATUS_EMOJI: Record = { - cancelled: "🚫", - completed: "✅", - failed: "❌", - queued: "🕒", - running: "▶️", +// Short ASCII tags: emoji are double-width in Discord's code blocks and wreck +// column alignment, which is the whole point of the tabular layout. +const STATUS_TAG: Record = { + cancelled: "cxl", + completed: "ok", + failed: "FAIL", + queued: "que", + running: "run", }; -export function formatStatus(report: StatusReport): string { - const lines: string[] = []; +const TAG_WIDTH = 5; +const THREAD_WIDTH = 29; +const AGE_WIDTH = 4; +const DUR_WIDTH = 5; +const ERROR_LINE_CHARS = 60; +/** + * Discord has no table markup; the closest thing is a monospace code block + * with hand-padded columns. Header line stays OUTSIDE the block (bold + emoji + * work there); rows stay ~45 chars wide so portrait mobile doesn't wrap. + */ +export function formatStatus(report: StatusReport): string { const mark = (value: boolean | null): string => value === null ? "❓" : value ? "✅" : "❌"; - lines.push( + const header = `**gerard status** · api-rs ${mark(report.apiHealthy)} ` + - `ready ${mark(report.apiReady)} · db ${report.dbOk ? "✅" : "❌"}`, - ); + `ready ${mark(report.apiReady)} · db ${report.dbOk ? "✅" : "❌"}`; + + const lines: string[] = []; const tallyEntries = Object.entries(report.tally).sort(); if (tallyEntries.length > 0) { lines.push( - `last 24h: ${tallyEntries - .map(([status, count]) => `${count} ${STATUS_EMOJI[status] ?? status}`) + `24h: ${tallyEntries + .map(([status, count]) => `${count} ${STATUS_TAG[status] ?? status}`) .join(" · ")}`, ); + lines.push(""); } - if (report.inFlight.length > 0) { - lines.push("in flight:"); - for (const row of report.inFlight) { - lines.push( - `${STATUS_EMOJI[row.status] ?? "•"} ${describeThread(row.threadKey)}` + - ` (${formatAge(row.ageSeconds)})`, - ); - } - } - - if (report.recent.length > 0) { - lines.push("recent turns:"); - for (const row of report.recent) { - const duration = - row.durationSeconds !== null - ? ` (${formatDuration(row.durationSeconds)})` - : ""; - const error = row.error ? ` — ${row.error}` : ""; - lines.push( - `${STATUS_EMOJI[row.status] ?? "•"} ${formatAge(row.ageSeconds)} ago · ` + - `${describeThread(row.threadKey)}${duration}${error}`, - ); + // One table: in-flight turns first (no duration yet), then settled recent + // turns. The recent query also returns queued/running rows — skip those so + // an in-flight turn isn't listed twice. + const turnRow = (row: ExecutionRow): string => { + const tag = (STATUS_TAG[row.status] ?? row.status).padEnd(TAG_WIDTH); + const thread = describeThread(row.threadKey).padEnd(THREAD_WIDTH); + const age = formatAge(row.ageSeconds).padStart(AGE_WIDTH); + const duration = ( + row.durationSeconds !== null ? formatDuration(row.durationSeconds) : "-" + ).padStart(DUR_WIDTH); + return `${tag} ${thread} ${age} ${duration}`.trimEnd(); + }; + const settled = report.recent.filter( + (row) => row.status !== "queued" && row.status !== "running", + ); + for (const row of [...report.inFlight, ...settled]) { + lines.push(turnRow(row)); + if (row.error) { + lines.push(` └ ${row.error.slice(0, ERROR_LINE_CHARS)}`); } } @@ -227,29 +237,40 @@ export function formatStatus(report: StatusReport): string { .join(", ")}`, ); } - if (sandboxBits.length > 0) lines.push(`sandboxes: ${sandboxBits.join(" · ")}`); + if (sandboxBits.length > 0) { + lines.push(""); + lines.push(`sandboxes: ${sandboxBits.join(" · ")}`); + } - for (const note of report.collectedNotes) lines.push(`⚠️ ${note}`); + for (const note of report.collectedNotes) lines.push(`! ${note}`); if (!report.dbOk && report.collectedNotes.length === 0) { - lines.push("⚠️ session database unreachable — turn history unavailable"); + lines.push("! session database unreachable — turn history unavailable"); } - const text = lines.join("\n"); - if (text.length <= STATUS_MAX_CHARS) return text; - return `${sliceSurrogateSafe(text, STATUS_MAX_CHARS - 12).trimEnd()}\n[truncated]`; + if (lines.length === 0) return header; + const body = lines.join("\n"); + const budget = STATUS_MAX_CHARS - header.length - 20; + const bounded = + body.length <= budget + ? body + : `${sliceSurrogateSafe(body, budget - 12).trimEnd()}\n[truncated]`; + return `${header}\n\`\`\`\n${bounded}\n\`\`\``; } -/** `platform · short thread name` from a session thread key. */ +/** `platform short-thread-name`, cut to the table's thread column width. */ function describeThread(threadKey: string): string { const separator = threadKey.indexOf(":"); - if (separator === -1) return shorten(threadKey); + if (separator === -1) return tailCut(threadKey, THREAD_WIDTH); const platform = threadKey.slice(0, separator); const rest = threadKey.slice(separator + 1); - return `${platform} ${shorten(rest)}`; + // Keep the platform readable and cut the rest from the front: thread keys + // front-load the constant part (guild/channel ids, owner/repo) and end with + // the discriminating bit. + return `${platform} ${tailCut(rest, THREAD_WIDTH - platform.length - 1)}`; } -function shorten(value: string): string { - return value.length <= 40 ? value : `…${value.slice(-39)}`; +function tailCut(value: string, width: number): string { + return value.length <= width ? value : `…${value.slice(-(width - 1))}`; } function formatAge(seconds: number | null): string { diff --git a/services/discordbot/test/status.test.ts b/services/discordbot/test/status.test.ts index dc01b2c5d..28b7e3579 100644 --- a/services/discordbot/test/status.test.ts +++ b/services/discordbot/test/status.test.ts @@ -172,7 +172,7 @@ describe("formatStatus", () => { warmPool: {}, }); - it("renders health, tallies, turns, and sandboxes compactly", async () => { + it("renders a code-block table with tags, tallies, and sandboxes", async () => { const report = await collectStatus({ apiUrl: "http://api", db: fullDb(), @@ -180,16 +180,42 @@ describe("formatStatus", () => { nowMs: NOW, }); const text = formatStatus(report); - expect(text).toContain("api-rs ✅"); - expect(text).toContain("41 ✅"); - expect(text).toContain("2 ❌"); - expect(text).toContain("github-manage 0xSplits/splits-teams:1799"); - expect(text).toContain("(1m)"); - expect(text).toContain("sandbox spawn timeout"); - expect(text).toContain("warm: 2 ready"); + // Header outside the block, data inside one fenced block. + expect(text.startsWith("**gerard status** · api-rs ✅")).toBe(true); + expect(text).toContain("```"); + expect(text).toContain("24h: 41 ok · 2 FAIL"); + // In-flight row first, no duration yet. + const lines = text.split("\n"); + const runLine = lines.find((line) => line.startsWith("run")); + expect(runLine).toContain("discord 1:2:3"); + expect(runLine?.trimEnd().endsWith("-")).toBe(true); + // Settled rows keep the platform on over-wide keys and carry age+duration. + expect(text).toContain("github-manage …"); + expect(text).toMatch(/ok\s+github-manage .*\s5m\s+1m/); + // Errors land on their own indented line. + expect(text).toContain("└ sandbox spawn timeout"); + expect(text).toContain("sandboxes: 1 active · warm: 2 ready"); expect(text.length).toBeLessThanOrEqual(2000); }); + it("keeps thread rows within the column budget", () => { + const report = baseReport(); + report.recent = [ + { + ageSeconds: 60, + durationSeconds: 30, + error: "", + status: "completed", + threadKey: `discord:${"9".repeat(60)}`, + }, + ]; + const text = formatStatus(report); + const row = text.split("\n").find((line) => line.startsWith("ok")); + expect(row).toBeDefined(); + expect(row).toContain("discord …"); + expect(row?.length ?? 0).toBeLessThanOrEqual(50); + }); + it("marks a down api-rs and unreachable DB honestly", () => { const report = baseReport(); report.apiHealthy = false; @@ -203,7 +229,7 @@ describe("formatStatus", () => { it("stays under the Discord cap with oversized errors", () => { const report = baseReport(); - report.recent = Array.from({ length: 12 }, (_, index) => ({ + report.recent = Array.from({ length: 30 }, (_, index) => ({ ageSeconds: 60 * index, durationSeconds: 5, error: "x".repeat(150), @@ -212,6 +238,7 @@ describe("formatStatus", () => { })); const text = formatStatus(report); expect(text.length).toBeLessThanOrEqual(2000); - expect(text.endsWith("[truncated]")).toBe(true); + expect(text.endsWith("```")).toBe(true); + expect(text).toContain("[truncated]"); }); }); From 2b516456c966cee0528940987d2daf2a481b7420 Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 18:26:47 +0200 Subject: [PATCH 03/10] feat(discordbot): status table headings, requester column, friendly labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Timing columns were ambiguous — add a THREAD/WHO/AGE/TOOK heading row (AGE = when requested, TOOK = runtime). Rows now prefer the session title (the conversation name the bots already set) over raw thread keys, fall back to a friendly rendering for untitled management turns ("GH PR splits-teams#1799"), and carry the requester from the execute metadata's user_name. Truncation keeps both ends readable (middle ellipsis) except names, which head-cut. Co-Authored-By: Claude Fable 5 --- services/discordbot/src/status.ts | 124 ++++++++++++++++-------- services/discordbot/test/status.test.ts | 30 ++++-- 2 files changed, 109 insertions(+), 45 deletions(-) diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts index 33d07969d..9f871483f 100644 --- a/services/discordbot/src/status.ts +++ b/services/discordbot/src/status.ts @@ -38,6 +38,10 @@ export type ExecutionRow = { error: string; status: string; threadKey: string; + /** Session title (the conversation name the bots set), when present. */ + title: string; + /** Display name of whoever triggered the turn, when recorded. */ + who: string; }; export type StatusReport = { @@ -95,6 +99,8 @@ export async function collectStatus(input: { error: String(row.error ?? "").slice(0, ERROR_SNIPPET_CHARS), status: String(row.status ?? "unknown"), threadKey: String(row.thread_key ?? "?"), + title: String(row.title ?? ""), + who: String(row.user_name ?? ""), }); const [apiHealthy, apiReady, recent, tally, inFlight, sandboxes, warm] = @@ -103,11 +109,15 @@ export async function collectStatus(input: { probe("/readyz"), query( "recent turns", - `SELECT thread_key, status, left(coalesce(error, ''), ${ERROR_SNIPPET_CHARS}) AS error, - created_at, - extract(epoch FROM (completed_at - started_at)) AS duration_seconds - FROM session_executions - ORDER BY created_at DESC + `SELECT e.thread_key, e.status, + left(coalesce(e.error, ''), ${ERROR_SNIPPET_CHARS}) AS error, + e.created_at, + extract(epoch FROM (e.completed_at - e.started_at)) AS duration_seconds, + e.metadata ->> 'user_name' AS user_name, + s.title + FROM session_executions e + LEFT JOIN sessions s ON s.thread_key = e.thread_key + ORDER BY e.created_at DESC LIMIT 8`, ), query( @@ -119,11 +129,14 @@ export async function collectStatus(input: { ), query( "in-flight turns", - `SELECT thread_key, status, '' AS error, created_at, - NULL AS duration_seconds - FROM session_executions - WHERE status IN ('queued', 'running') - ORDER BY created_at ASC + `SELECT e.thread_key, e.status, '' AS error, e.created_at, + NULL AS duration_seconds, + e.metadata ->> 'user_name' AS user_name, + s.title + FROM session_executions e + LEFT JOIN sessions s ON s.thread_key = e.thread_key + WHERE e.status IN ('queued', 'running') + ORDER BY e.created_at ASC LIMIT 8`, ), query( @@ -174,7 +187,8 @@ const STATUS_TAG: Record = { }; const TAG_WIDTH = 5; -const THREAD_WIDTH = 29; +const THREAD_WIDTH = 24; +const WHO_WIDTH = 8; const AGE_WIDTH = 4; const DUR_WIDTH = 5; const ERROR_LINE_CHARS = 60; @@ -182,7 +196,7 @@ const ERROR_LINE_CHARS = 60; /** * Discord has no table markup; the closest thing is a monospace code block * with hand-padded columns. Header line stays OUTSIDE the block (bold + emoji - * work there); rows stay ~45 chars wide so portrait mobile doesn't wrap. + * work there); rows stay ~50 chars wide to limit wrapping on mobile. */ export function formatStatus(report: StatusReport): string { const mark = (value: boolean | null): string => @@ -203,25 +217,40 @@ export function formatStatus(report: StatusReport): string { lines.push(""); } + const tableRow = ( + tag: string, + thread: string, + who: string, + age: string, + took: string, + ): string => + `${tag.padEnd(TAG_WIDTH)} ${fit(thread, THREAD_WIDTH)} ` + + `${fit(who, WHO_WIDTH, "head")} ${age.padStart(AGE_WIDTH)} ` + + `${took.padStart(DUR_WIDTH)}`; + // One table: in-flight turns first (no duration yet), then settled recent // turns. The recent query also returns queued/running rows — skip those so - // an in-flight turn isn't listed twice. - const turnRow = (row: ExecutionRow): string => { - const tag = (STATUS_TAG[row.status] ?? row.status).padEnd(TAG_WIDTH); - const thread = describeThread(row.threadKey).padEnd(THREAD_WIDTH); - const age = formatAge(row.ageSeconds).padStart(AGE_WIDTH); - const duration = ( - row.durationSeconds !== null ? formatDuration(row.durationSeconds) : "-" - ).padStart(DUR_WIDTH); - return `${tag} ${thread} ${age} ${duration}`.trimEnd(); - }; + // an in-flight turn isn't listed twice. AGE = when the turn was requested, + // TOOK = how long it ran. + const turnRow = (row: ExecutionRow): string => + tableRow( + STATUS_TAG[row.status] ?? row.status, + threadLabel(row), + row.who, + formatAge(row.ageSeconds), + row.durationSeconds !== null ? formatDuration(row.durationSeconds) : "-", + ).trimEnd(); const settled = report.recent.filter( (row) => row.status !== "queued" && row.status !== "running", ); - for (const row of [...report.inFlight, ...settled]) { - lines.push(turnRow(row)); - if (row.error) { - lines.push(` └ ${row.error.slice(0, ERROR_LINE_CHARS)}`); + const turns = [...report.inFlight, ...settled]; + if (turns.length > 0) { + lines.push(tableRow("", "THREAD", "WHO", "AGE", "TOOK").trimEnd()); + for (const row of turns) { + lines.push(turnRow(row)); + if (row.error) { + lines.push(` └ ${row.error.slice(0, ERROR_LINE_CHARS)}`); + } } } @@ -257,20 +286,39 @@ export function formatStatus(report: StatusReport): string { return `${header}\n\`\`\`\n${bounded}\n\`\`\``; } -/** `platform short-thread-name`, cut to the table's thread column width. */ -function describeThread(threadKey: string): string { - const separator = threadKey.indexOf(":"); - if (separator === -1) return tailCut(threadKey, THREAD_WIDTH); - const platform = threadKey.slice(0, separator); - const rest = threadKey.slice(separator + 1); - // Keep the platform readable and cut the rest from the front: thread keys - // front-load the constant part (guild/channel ids, owner/repo) and end with - // the discriminating bit. - return `${platform} ${tailCut(rest, THREAD_WIDTH - platform.length - 1)}`; +/** + * Human label for a turn: the session title when the bots set one, otherwise + * a friendlier rendering of the thread key ("GH PR splits-teams#1799" beats + * "github-manage:0xSplits/splits-teams:1799"; raw Discord ids stay raw). + */ +function threadLabel(row: { threadKey: string; title: string }): string { + if (row.title.trim()) return row.title.trim(); + const parts = row.threadKey.split(":"); + const platform = parts[0] ?? row.threadKey; + const rest = parts.slice(1).join(":"); + if (platform === "github-manage" && parts.length >= 3) { + const repo = (parts[1] ?? "").split("/").pop() ?? parts[1]; + return `GH PR ${repo}#${parts[2]}`; + } + if (platform.startsWith("github")) return `GH ${rest}`; + if (platform === "linear") return `Linear ${rest}`; + if (platform === "slack") return `Slack ${rest}`; + if (platform === "discord") return `Discord ${rest}`; + return row.threadKey; } -function tailCut(value: string, width: number): string { - return value.length <= width ? value : `…${value.slice(-(width - 1))}`; +/** + * Truncate + pad to the column. Middle ellipsis by default so both ends stay + * readable ("GH PR splits-con…eams#1799", "Discord 90294…:1391220231" — the + * head names the thing, the tail discriminates); plain head-cut for names. + */ +function fit(value: string, width: number, keep: "edges" | "head" = "edges"): string { + if (value.length <= width) return value.padEnd(width); + if (keep === "head" || width < 12) { + return `${value.slice(0, width - 1)}…`; + } + const tail = 7; + return `${value.slice(0, width - tail - 1)}…${value.slice(-tail)}`; } function formatAge(seconds: number | null): string { diff --git a/services/discordbot/test/status.test.ts b/services/discordbot/test/status.test.ts index 28b7e3579..8775d32de 100644 --- a/services/discordbot/test/status.test.ts +++ b/services/discordbot/test/status.test.ts @@ -60,6 +60,8 @@ function fullDb(): StatusDb { error: "", status: "running", thread_key: "discord:1:2:3", + title: "fix the deploy pipeline", + user_name: "oliver", }, ]; } @@ -71,6 +73,8 @@ function fullDb(): StatusDb { error: "", status: "completed", thread_key: "github-manage:0xSplits/splits-teams:1799", + title: null, + user_name: "0xdiid", }, { created_at: new Date(NOW - 1_900_000), @@ -78,6 +82,8 @@ function fullDb(): StatusDb { error: "sandbox spawn timeout after 120s", status: "failed", thread_key: "discord:1:2:9", + title: null, + user_name: "jaan", }, ]; } @@ -184,14 +190,17 @@ describe("formatStatus", () => { expect(text.startsWith("**gerard status** · api-rs ✅")).toBe(true); expect(text).toContain("```"); expect(text).toContain("24h: 41 ok · 2 FAIL"); - // In-flight row first, no duration yet. + // Column headings above the turn table. + expect(text).toMatch(/THREAD\s+WHO\s+AGE\s+TOOK/); + // In-flight row first: session title, requester, no duration yet. const lines = text.split("\n"); const runLine = lines.find((line) => line.startsWith("run")); - expect(runLine).toContain("discord 1:2:3"); + expect(runLine).toContain("fix the deploy pipeline"); + expect(runLine).toContain("oliver"); expect(runLine?.trimEnd().endsWith("-")).toBe(true); - // Settled rows keep the platform on over-wide keys and carry age+duration. - expect(text).toContain("github-manage …"); - expect(text).toMatch(/ok\s+github-manage .*\s5m\s+1m/); + // Untitled management turn falls back to the friendly PR label. + expect(text).toContain("GH PR splits-teams#1799"); + expect(text).toMatch(/ok\s+GH PR splits-teams#1799\s+0xdiid\s+5m\s+1m/); // Errors land on their own indented line. expect(text).toContain("└ sandbox spawn timeout"); expect(text).toContain("sandboxes: 1 active · warm: 2 ready"); @@ -207,13 +216,18 @@ describe("formatStatus", () => { error: "", status: "completed", threadKey: `discord:${"9".repeat(60)}`, + title: "", + who: "someone-with-a-long-name", }, ]; const text = formatStatus(report); const row = text.split("\n").find((line) => line.startsWith("ok")); expect(row).toBeDefined(); - expect(row).toContain("discord …"); - expect(row?.length ?? 0).toBeLessThanOrEqual(50); + // Middle ellipsis keeps the platform head and the id tail. + expect(row).toContain("Discord 9"); + expect(row).toContain("…"); + expect(row).toContain("someone…"); + expect(row?.length ?? 0).toBeLessThanOrEqual(52); }); it("marks a down api-rs and unreachable DB honestly", () => { @@ -235,6 +249,8 @@ describe("formatStatus", () => { error: "x".repeat(150), status: "failed", threadKey: `discord:${"y".repeat(80)}:${index}`, + title: "", + who: "someone", })); const text = formatStatus(report); expect(text.length).toBeLessThanOrEqual(2000); From d8219f0642ace5347038390b6f87eb68700eff98 Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 20:25:23 +0200 Subject: [PATCH 04/10] fix(discordbot): honest warm-pool numbers, discord titles, wider WHO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claimed/failed warm-sandbox rows are never deleted — an unfiltered count reads like a leak ("868 claimed"). Window them to 24h and split the line into current pool (ready/evicting) vs churn. Discord sessions store their conversation name in metadata (discord_conversation_name), not title — coalesce it in so Discord rows show thread names instead of raw ids. Widen WHO to 10 and alias github-pr-manager to gerard. Co-Authored-By: Claude Fable 5 --- services/discordbot/src/status.ts | 37 +++++++++++++++++-------- services/discordbot/test/status.test.ts | 17 ++++++++---- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts index 9f871483f..bed7eb526 100644 --- a/services/discordbot/src/status.ts +++ b/services/discordbot/src/status.ts @@ -114,7 +114,7 @@ export async function collectStatus(input: { e.created_at, extract(epoch FROM (e.completed_at - e.started_at)) AS duration_seconds, e.metadata ->> 'user_name' AS user_name, - s.title + coalesce(s.title, s.metadata ->> 'discord_conversation_name') AS title FROM session_executions e LEFT JOIN sessions s ON s.thread_key = e.thread_key ORDER BY e.created_at DESC @@ -132,7 +132,7 @@ export async function collectStatus(input: { `SELECT e.thread_key, e.status, '' AS error, e.created_at, NULL AS duration_seconds, e.metadata ->> 'user_name' AS user_name, - s.title + coalesce(s.title, s.metadata ->> 'discord_conversation_name') AS title FROM session_executions e LEFT JOIN sessions s ON s.thread_key = e.thread_key WHERE e.status IN ('queued', 'running') @@ -148,10 +148,15 @@ export async function collectStatus(input: { ORDER BY sandbox_last_active_at DESC LIMIT 8`, ), + // Claimed/failed rows are never deleted — they're lifetime history, so + // an unfiltered count reads like a leak ("868 claimed"). Only ready/ + // evicting are current facts; show claimed/failed as 24h churn. query( "warm pool", `SELECT status, count(*)::int AS count FROM session_warm_sandboxes + WHERE status IN ('ready', 'evicting') + OR updated_at > now() - interval '24 hours' GROUP BY status`, ), ]); @@ -188,7 +193,12 @@ const STATUS_TAG: Record = { const TAG_WIDTH = 5; const THREAD_WIDTH = 24; -const WHO_WIDTH = 8; +const WHO_WIDTH = 10; + +// Internal actor ids nobody recognizes → the name the team knows. +const WHO_ALIAS: Record = { + "github-pr-manager": "gerard", +}; const AGE_WIDTH = 4; const DUR_WIDTH = 5; const ERROR_LINE_CHARS = 60; @@ -236,7 +246,7 @@ export function formatStatus(report: StatusReport): string { tableRow( STATUS_TAG[row.status] ?? row.status, threadLabel(row), - row.who, + WHO_ALIAS[row.who] ?? row.who, formatAge(row.ageSeconds), row.durationSeconds !== null ? formatDuration(row.durationSeconds) : "-", ).trimEnd(); @@ -254,18 +264,21 @@ export function formatStatus(report: StatusReport): string { } } - const warmEntries = Object.entries(report.warmPool).sort(); const sandboxBits: string[] = []; if (report.sandboxes.length > 0) { sandboxBits.push(`${report.sandboxes.length} active`); } - if (warmEntries.length > 0) { - sandboxBits.push( - `warm: ${warmEntries - .map(([status, count]) => `${count} ${status}`) - .join(", ")}`, - ); - } + // ready/evicting are the pool's current state; claimed/failed rows are + // historical (the collect query already windows them to 24h). + const warmLine = (statuses: string[]): string => + statuses + .filter((status) => (report.warmPool[status] ?? 0) > 0) + .map((status) => `${report.warmPool[status]} ${status}`) + .join(", "); + const warmNow = warmLine(["ready", "evicting"]); + const warmChurn = warmLine(["claimed", "failed"]); + if (warmNow) sandboxBits.push(`warm: ${warmNow}`); + if (warmChurn) sandboxBits.push(`warm 24h: ${warmChurn}`); if (sandboxBits.length > 0) { lines.push(""); lines.push(`sandboxes: ${sandboxBits.join(" · ")}`); diff --git a/services/discordbot/test/status.test.ts b/services/discordbot/test/status.test.ts index 8775d32de..c079686d2 100644 --- a/services/discordbot/test/status.test.ts +++ b/services/discordbot/test/status.test.ts @@ -46,6 +46,12 @@ const NOW = Date.parse("2026-08-12T12:00:00Z"); function fullDb(): StatusDb { return stubDb((sql) => { + if (sql.includes("session_warm_sandboxes")) { + return [ + { status: "ready", count: 2 }, + { status: "claimed", count: 41 }, + ]; + } if (sql.includes("interval '24 hours'")) { return [ { status: "completed", count: 41 }, @@ -96,9 +102,6 @@ function fullDb(): StatusDb { }, ]; } - if (sql.includes("session_warm_sandboxes")) { - return [{ status: "ready", count: 2 }]; - } return []; }); } @@ -119,7 +122,7 @@ describe("collectStatus", () => { expect(report.recent[1]?.error).toContain("sandbox spawn timeout"); expect(report.inFlight).toHaveLength(1); expect(report.sandboxes[0]?.sandboxId).toBe("asbx-1755000000-1"); - expect(report.warmPool).toEqual({ ready: 2 }); + expect(report.warmPool).toEqual({ claimed: 41, ready: 2 }); expect(report.collectedNotes).toEqual([]); }); @@ -203,7 +206,9 @@ describe("formatStatus", () => { expect(text).toMatch(/ok\s+GH PR splits-teams#1799\s+0xdiid\s+5m\s+1m/); // Errors land on their own indented line. expect(text).toContain("└ sandbox spawn timeout"); - expect(text).toContain("sandboxes: 1 active · warm: 2 ready"); + expect(text).toContain( + "sandboxes: 1 active · warm: 2 ready · warm 24h: 41 claimed", + ); expect(text.length).toBeLessThanOrEqual(2000); }); @@ -226,7 +231,7 @@ describe("formatStatus", () => { // Middle ellipsis keeps the platform head and the id tail. expect(row).toContain("Discord 9"); expect(row).toContain("…"); - expect(row).toContain("someone…"); + expect(row).toContain("someone-w…"); expect(row?.length ?? 0).toBeLessThanOrEqual(52); }); From 5bd2ecea78706c9305d0e52e30060b5ebe4641d5 Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 20:30:07 +0200 Subject: [PATCH 05/10] fix(discordbot): pick up linear/slack conversation names for status rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linear and Slack sessions store their conversation names under their own metadata keys (linear_conversation_name / slack_conversation_name), same pattern as discord — fold them into the title coalesce so those rows show issue/channel names instead of raw UUIDs. Co-Authored-By: Claude Fable 5 --- services/discordbot/src/status.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts index bed7eb526..afd424f71 100644 --- a/services/discordbot/src/status.ts +++ b/services/discordbot/src/status.ts @@ -114,7 +114,9 @@ export async function collectStatus(input: { e.created_at, extract(epoch FROM (e.completed_at - e.started_at)) AS duration_seconds, e.metadata ->> 'user_name' AS user_name, - coalesce(s.title, s.metadata ->> 'discord_conversation_name') AS title + coalesce(s.title, s.metadata ->> 'discord_conversation_name', + s.metadata ->> 'linear_conversation_name', + s.metadata ->> 'slack_conversation_name') AS title FROM session_executions e LEFT JOIN sessions s ON s.thread_key = e.thread_key ORDER BY e.created_at DESC @@ -132,7 +134,9 @@ export async function collectStatus(input: { `SELECT e.thread_key, e.status, '' AS error, e.created_at, NULL AS duration_seconds, e.metadata ->> 'user_name' AS user_name, - coalesce(s.title, s.metadata ->> 'discord_conversation_name') AS title + coalesce(s.title, s.metadata ->> 'discord_conversation_name', + s.metadata ->> 'linear_conversation_name', + s.metadata ->> 'slack_conversation_name') AS title FROM session_executions e LEFT JOIN sessions s ON s.thread_key = e.thread_key WHERE e.status IN ('queued', 'running') From 667845d85c42e422685902d8e6fda789cf159749 Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 20:36:56 +0200 Subject: [PATCH 06/10] feat(discordbot): 7-day run histogram + failure rate in status reply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unicode bar chart per UTC day (zero-filled week, bars scaled to the busiest day), with failures as their own labeled column — one measure per bar, no second scale, no color-alone signal — and a 7d totals/failure-rate stat line beneath. Co-Authored-By: Claude Fable 5 --- services/discordbot/src/status.ts | 79 ++++++++++++++++++++++++- services/discordbot/test/status.test.ts | 20 +++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts index afd424f71..f3b229c75 100644 --- a/services/discordbot/src/status.ts +++ b/services/discordbot/src/status.ts @@ -44,10 +44,19 @@ export type ExecutionRow = { who: string; }; +export type DailyRow = { + /** UTC calendar date, `YYYY-MM-DD`. */ + day: string; + failed: number; + runs: number; +}; + export type StatusReport = { apiHealthy: boolean | null; apiReady: boolean | null; collectedNotes: string[]; + /** Runs per UTC day, oldest→today, zero-filled to exactly 7 entries. */ + daily: DailyRow[]; dbOk: boolean; inFlight: ExecutionRow[]; recent: ExecutionRow[]; @@ -103,7 +112,7 @@ export async function collectStatus(input: { who: String(row.user_name ?? ""), }); - const [apiHealthy, apiReady, recent, tally, inFlight, sandboxes, warm] = + const [apiHealthy, apiReady, recent, tally, inFlight, sandboxes, warm, daily] = await Promise.all([ probe("/healthz"), probe("/readyz"), @@ -163,12 +172,23 @@ export async function collectStatus(input: { OR updated_at > now() - interval '24 hours' GROUP BY status`, ), + query( + "7-day histogram", + `SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS day, + count(*)::int AS runs, + (count(*) FILTER (WHERE status = 'failed'))::int AS failed + FROM session_executions + WHERE created_at > now() - interval '7 days' + GROUP BY 1 + ORDER BY 1`, + ), ]); return { apiHealthy, apiReady, collectedNotes: notes, + daily: zeroFilledWeek(daily ?? [], now), dbOk: recent !== null, inFlight: (inFlight ?? []).map(toExecutionRow), recent: (recent ?? []).map(toExecutionRow), @@ -206,6 +226,15 @@ const WHO_ALIAS: Record = { const AGE_WIDTH = 4; const DUR_WIDTH = 5; const ERROR_LINE_CHARS = 60; +const BAR_WIDTH = 16; + +const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; + +function weekdayLabel(dayIso: string): string { + const parsed = new Date(`${dayIso}T00:00:00Z`); + const label = WEEKDAYS[parsed.getUTCDay()]; + return label ?? "???"; +} /** * Discord has no table markup; the closest thing is a monospace code block @@ -231,6 +260,32 @@ export function formatStatus(report: StatusReport): string { lines.push(""); } + // 7-day histogram: the bar encodes ONE measure (runs); failures get their + // own labeled column rather than a second scale or color-alone marking, and + // the failure rate is a plain stat line. + const week = report.daily; + const totalRuns = week.reduce((sum, day) => sum + day.runs, 0); + if (totalRuns > 0) { + const totalFailed = week.reduce((sum, day) => sum + day.failed, 0); + const maxRuns = Math.max(...week.map((day) => day.runs)); + lines.push(` ${"LAST 7 DAYS".padEnd(BAR_WIDTH + 1)}RUNS FAIL`); + for (const day of week) { + const bar = "█".repeat( + day.runs === 0 ? 0 : Math.max(1, Math.round((day.runs / maxRuns) * BAR_WIDTH)), + ); + const fail = day.failed > 0 ? String(day.failed) : "-"; + lines.push( + `${weekdayLabel(day.day)} ${bar.padEnd(BAR_WIDTH + 1)}` + + `${String(day.runs).padStart(4)} ${fail.padStart(4)}`, + ); + } + const rate = totalRuns > 0 ? (totalFailed / totalRuns) * 100 : 0; + lines.push( + `7d: ${totalRuns} runs · ${totalFailed} failed (${rate.toFixed(1)}%)`, + ); + lines.push(""); + } + const tableRow = ( tag: string, thread: string, @@ -351,6 +406,28 @@ function formatDuration(seconds: number): string { return `${Math.round(s / 86400)}d`; } +/** The last 7 UTC calendar days (oldest→today), zero-filling days with no runs. */ +function zeroFilledWeek( + rows: Record[], + nowMs: number, +): DailyRow[] { + const byDay = new Map(); + for (const row of rows) { + byDay.set(String(row.day ?? ""), { + failed: numberOrNull(row.failed) ?? 0, + runs: numberOrNull(row.runs) ?? 0, + }); + } + const days: DailyRow[] = []; + for (let offset = 6; offset >= 0; offset -= 1) { + const day = new Date(nowMs - offset * 86_400_000) + .toISOString() + .slice(0, 10); + days.push({ day, failed: 0, runs: 0, ...byDay.get(day) }); + } + return days; +} + function countsByStatus( rows: Record[] | null, ): Record { diff --git a/services/discordbot/test/status.test.ts b/services/discordbot/test/status.test.ts index c079686d2..c9c674490 100644 --- a/services/discordbot/test/status.test.ts +++ b/services/discordbot/test/status.test.ts @@ -52,6 +52,13 @@ function fullDb(): StatusDb { { status: "claimed", count: 41 }, ]; } + if (sql.includes("interval '7 days'")) { + return [ + { day: "2026-08-10", failed: 0, runs: 12 }, + { day: "2026-08-11", failed: 3, runs: 40 }, + { day: "2026-08-12", failed: 0, runs: 20 }, + ]; + } if (sql.includes("interval '24 hours'")) { return [ { status: "completed", count: 41 }, @@ -124,6 +131,11 @@ describe("collectStatus", () => { expect(report.sandboxes[0]?.sandboxId).toBe("asbx-1755000000-1"); expect(report.warmPool).toEqual({ claimed: 41, ready: 2 }); expect(report.collectedNotes).toEqual([]); + // Zero-filled to exactly 7 UTC days, oldest first, today last. + expect(report.daily).toHaveLength(7); + expect(report.daily[0]).toEqual({ day: "2026-08-06", failed: 0, runs: 0 }); + expect(report.daily[5]).toEqual({ day: "2026-08-11", failed: 3, runs: 40 }); + expect(report.daily[6]).toEqual({ day: "2026-08-12", failed: 0, runs: 20 }); }); it("still reports DB data when api-rs is down", async () => { @@ -173,6 +185,7 @@ describe("formatStatus", () => { apiHealthy: true, apiReady: true, collectedNotes: [], + daily: [], dbOk: true, inFlight: [], recent: [], @@ -195,6 +208,13 @@ describe("formatStatus", () => { expect(text).toContain("24h: 41 ok · 2 FAIL"); // Column headings above the turn table. expect(text).toMatch(/THREAD\s+WHO\s+AGE\s+TOOK/); + // 7-day histogram: full-width bar on the busiest day, "-" for zero + // failures, zero-run days barless, and a failure-rate stat line. + expect(text).toMatch(/LAST 7 DAYS\s+RUNS FAIL/); + expect(text).toMatch(/Tue {2}█{16}\s+40\s+3/); + expect(text).toMatch(/Wed {2}█+\s+20\s+-/); + expect(text).toMatch(/Thu {2}\s+0\s+-/); + expect(text).toContain("7d: 72 runs · 3 failed (4.2%)"); // In-flight row first: session title, requester, no duration yet. const lines = text.split("\n"); const runLine = lines.find((line) => line.startsWith("run")); From 73e522dda0352a95e423f6a2f4720f4c769bb693 Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 20:40:36 +0200 Subject: [PATCH 07/10] fix(discordbot): histogram in its own code block below the live view Keep the first block exactly as it was (tally, turn table, sandboxes) and append the 7-day histogram as a second fenced block, budgeting the live view's truncation around the fixed-size histogram. Co-Authored-By: Claude Fable 5 --- services/discordbot/src/status.ts | 67 ++++++++++++++----------- services/discordbot/test/status.test.ts | 9 +++- 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts index f3b229c75..baf1c8b9e 100644 --- a/services/discordbot/src/status.ts +++ b/services/discordbot/src/status.ts @@ -260,32 +260,6 @@ export function formatStatus(report: StatusReport): string { lines.push(""); } - // 7-day histogram: the bar encodes ONE measure (runs); failures get their - // own labeled column rather than a second scale or color-alone marking, and - // the failure rate is a plain stat line. - const week = report.daily; - const totalRuns = week.reduce((sum, day) => sum + day.runs, 0); - if (totalRuns > 0) { - const totalFailed = week.reduce((sum, day) => sum + day.failed, 0); - const maxRuns = Math.max(...week.map((day) => day.runs)); - lines.push(` ${"LAST 7 DAYS".padEnd(BAR_WIDTH + 1)}RUNS FAIL`); - for (const day of week) { - const bar = "█".repeat( - day.runs === 0 ? 0 : Math.max(1, Math.round((day.runs / maxRuns) * BAR_WIDTH)), - ); - const fail = day.failed > 0 ? String(day.failed) : "-"; - lines.push( - `${weekdayLabel(day.day)} ${bar.padEnd(BAR_WIDTH + 1)}` + - `${String(day.runs).padStart(4)} ${fail.padStart(4)}`, - ); - } - const rate = totalRuns > 0 ? (totalFailed / totalRuns) * 100 : 0; - lines.push( - `7d: ${totalRuns} runs · ${totalFailed} failed (${rate.toFixed(1)}%)`, - ); - lines.push(""); - } - const tableRow = ( tag: string, thread: string, @@ -348,14 +322,49 @@ export function formatStatus(report: StatusReport): string { lines.push("! session database unreachable — turn history unavailable"); } - if (lines.length === 0) return header; + // 7-day histogram, in its OWN code block below the live view: the bar + // encodes ONE measure (runs); failures get their own labeled column rather + // than a second scale or color-alone marking; the failure rate is a plain + // stat line. + const histogramLines: string[] = []; + const week = report.daily; + const totalRuns = week.reduce((sum, day) => sum + day.runs, 0); + if (totalRuns > 0) { + const totalFailed = week.reduce((sum, day) => sum + day.failed, 0); + const maxRuns = Math.max(...week.map((day) => day.runs)); + histogramLines.push(` ${"LAST 7 DAYS".padEnd(BAR_WIDTH + 1)}RUNS FAIL`); + for (const day of week) { + const bar = "█".repeat( + day.runs === 0 + ? 0 + : Math.max(1, Math.round((day.runs / maxRuns) * BAR_WIDTH)), + ); + const fail = day.failed > 0 ? String(day.failed) : "-"; + histogramLines.push( + `${weekdayLabel(day.day)} ${bar.padEnd(BAR_WIDTH + 1)}` + + `${String(day.runs).padStart(4)} ${fail.padStart(4)}`, + ); + } + const rate = (totalFailed / totalRuns) * 100; + histogramLines.push( + `7d: ${totalRuns} runs · ${totalFailed} failed (${rate.toFixed(1)}%)`, + ); + } + const histogram = histogramLines.join("\n"); + const histogramBlock = histogram ? `\n\`\`\`\n${histogram}\n\`\`\`` : ""; + + if (lines.length === 0 && !histogramBlock) return header; const body = lines.join("\n"); - const budget = STATUS_MAX_CHARS - header.length - 20; + // The histogram block is small and fixed-size; give the live view whatever + // budget remains under Discord's cap. + const budget = + STATUS_MAX_CHARS - header.length - histogramBlock.length - 20; const bounded = body.length <= budget ? body : `${sliceSurrogateSafe(body, budget - 12).trimEnd()}\n[truncated]`; - return `${header}\n\`\`\`\n${bounded}\n\`\`\``; + const liveBlock = lines.length > 0 ? `\n\`\`\`\n${bounded}\n\`\`\`` : ""; + return `${header}${liveBlock}${histogramBlock}`; } /** diff --git a/services/discordbot/test/status.test.ts b/services/discordbot/test/status.test.ts index c9c674490..86441cc20 100644 --- a/services/discordbot/test/status.test.ts +++ b/services/discordbot/test/status.test.ts @@ -208,8 +208,13 @@ describe("formatStatus", () => { expect(text).toContain("24h: 41 ok · 2 FAIL"); // Column headings above the turn table. expect(text).toMatch(/THREAD\s+WHO\s+AGE\s+TOOK/); - // 7-day histogram: full-width bar on the busiest day, "-" for zero - // failures, zero-run days barless, and a failure-rate stat line. + // 7-day histogram in its OWN code block, after the live view: full-width + // bar on the busiest day, "-" for zero failures, zero-run days barless, + // and a failure-rate stat line. + expect(text.split("```")).toHaveLength(5); + expect(text.indexOf("LAST 7 DAYS")).toBeGreaterThan( + text.indexOf("sandboxes:"), + ); expect(text).toMatch(/LAST 7 DAYS\s+RUNS FAIL/); expect(text).toMatch(/Tue {2}█{16}\s+40\s+3/); expect(text).toMatch(/Wed {2}█+\s+20\s+-/); From 24abf2b33497ae451d4367b4a361a3a77bfd9f64 Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 20:46:00 +0200 Subject: [PATCH 08/10] fix(discordbot): label warm-pool failures as spawn-failed "warm 24h: 3 failed" next to a histogram FAIL column reads like failed turns; it's warm-sandbox provisioning failures (the next session just cold-starts). Rename the display label so the two failure domains can't be conflated. Co-Authored-By: Claude Fable 5 --- services/discordbot/src/status.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts index baf1c8b9e..506f4e5cb 100644 --- a/services/discordbot/src/status.ts +++ b/services/discordbot/src/status.ts @@ -302,11 +302,17 @@ export function formatStatus(report: StatusReport): string { sandboxBits.push(`${report.sandboxes.length} active`); } // ready/evicting are the pool's current state; claimed/failed rows are - // historical (the collect query already windows them to 24h). + // historical (the collect query already windows them to 24h). "failed" here + // is a warm SPAWN failure (a standby sandbox that didn't provision — the + // next session cold-starts instead), NOT a failed turn; label it so it + // can't be confused with the histogram's FAIL column. + const WARM_LABEL: Record = { failed: "spawn-failed" }; const warmLine = (statuses: string[]): string => statuses .filter((status) => (report.warmPool[status] ?? 0) > 0) - .map((status) => `${report.warmPool[status]} ${status}`) + .map( + (status) => `${report.warmPool[status]} ${WARM_LABEL[status] ?? status}`, + ) .join(", "); const warmNow = warmLine(["ready", "evicting"]); const warmChurn = warmLine(["claimed", "failed"]); From 51cb2aaf8ed71605cb86afd29ddb1713c26f2b1e Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 21:49:42 +0200 Subject: [PATCH 09/10] feat(discordbot): status via api-rs endpoint, channel opt-in, hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework of the status fast-path after an adversarial review: - Data now comes from api-rs's read-only /api/status over HTTP with timeouts — no SQL from the ingress bot, no bot-side DB pool to hang the per-thread handler lock, and the service-ownership boundary holds. - The feature is opt-in per channel (DISCORDBOT_STATUS_CHANNEL_IDS; unset = disabled): the report exposes cross-platform activity, so where it may be seen is a deployment decision, not a default. - Header names the configured bot userName (no hardcoded deployment name); the actor alias map is gone. - Backticks/newlines in interpolated titles/errors are neutralized so a hostile title can't break out of the code block; failures post a generic line with internals kept to logs. - Mention-created threads are subscribed before the fast-path replies; unreachable (❓) is distinguished from unhealthy (❌). Co-Authored-By: Claude Fable 5 --- services/discordbot/src/index.ts | 59 ++--- services/discordbot/src/server.ts | 3 + services/discordbot/src/status.ts | 242 ++++++++------------- services/discordbot/src/types.ts | 7 + services/discordbot/test/status.test.ts | 275 +++++++++++++----------- 5 files changed, 280 insertions(+), 306 deletions(-) diff --git a/services/discordbot/src/index.ts b/services/discordbot/src/index.ts index 6b61ddb4b..18af11b52 100644 --- a/services/discordbot/src/index.ts +++ b/services/discordbot/src/index.ts @@ -33,7 +33,12 @@ import { renameThreadFromMessage, } from "./discord-threading"; import { setGatewayConnected } from "./gateway"; -import { collectStatus, formatStatus, isStatusCommand } from "./status"; +import { + STATUS_FAILURE_REPLY, + collectStatus, + formatStatus, + isStatusCommand, +} from "./status"; import { collectInitialContext, executeSessionTurn, @@ -260,47 +265,44 @@ export function createDiscordbot(options: DiscordbotOptions): Discordbot { DEFAULT_MAX_CONCURRENT_EXECUTIONS_PER_GUILD, ); - // Lazy pool for the status fast-path, separate from the state adapter's pool - // (which is private to createDefaultState — and options.state deployments - // have no pool at all). Capped small: status is an occasional human command. - let statusPool: pg.Pool | null = null; - const statusDb = (): pg.Pool | null => { - if (!options.postgresUrl) return null; - if (!statusPool) { - statusPool = new pg.Pool({ - connectionString: options.postgresUrl, - max: 2, - }); - statusPool.on("error", (error) => { - logger.warn("discordbot_status_pool_error", { - error: errorMessage(error), - }); - }); - } - return statusPool; - }; - // A bare "status"/"health" mention answers straight from the control plane - // (api-rs health + the shared session DB) with no sandbox turn — so it still - // works when the agent pipeline is what's broken. Returns true when handled. + // (api-rs /healthz + /api/status over HTTP — no SQL, no sandbox turn) so it + // still works when the agent pipeline is what's broken. The feature is + // OPT-IN per channel: statusChannelAllowlist empty/unset disables it + // entirely (the report exposes cross-platform activity — session titles, + // requester names, error snippets — so which channels may see it is a + // deployment decision, not a default). The exchange deliberately stays out + // of the session transcript: it's operational metadata, not conversation + // the agent should later see. Returns true when handled. + const statusChannels = new Set(options.statusChannelAllowlist ?? []); + const isStatusChannel = (threadKey: string): boolean => { + if (statusChannels.size === 0) return false; + const { channelId, threadId } = parseDiscordThreadKey(threadKey); + return ( + (channelId !== undefined && statusChannels.has(channelId)) || + (threadId !== undefined && statusChannels.has(threadId)) + ); + }; const maybeReplyStatus = async ( thread: Thread, message: ChatMessage, ): Promise => { if (!isStatusCommand(message.text ?? "")) return false; + if (!isStatusChannel(thread.id)) return false; try { const report = await collectStatus({ apiUrl: options.apiUrl, - db: statusDb(), fetchFn: options.fetch, }); - await thread.post(formatStatus(report)); + await thread.post(formatStatus(report, userName)); } catch (error) { + // Internals (hostnames, auth errors) stay in the logs; the channel gets + // a generic line. logger.warn("discordbot_status_reply_failed", { error: errorMessage(error), }); try { - await thread.post(`⚠️ status check failed: ${errorMessage(error)}`); + await thread.post(STATUS_FAILURE_REPLY); } catch { // best-effort; nothing left to signal with. } @@ -310,8 +312,11 @@ export function createDiscordbot(options: DiscordbotOptions): Discordbot { chat.onNewMention(async (thread, message) => { if (!isAllowedDiscordMessage(message, options, logger)) return; - if (await maybeReplyStatus(thread, message)) return; + // Subscribe before the status fast-path: the adapter has already created + // a thread for the mention, and an unsubscribed thread would silently + // ignore follow-ups. await thread.subscribe(); + if (await maybeReplyStatus(thread, message)) return; await syncThreadMessageToSession(thread, message, { executionLimiter, mode: "execute", diff --git a/services/discordbot/src/server.ts b/services/discordbot/src/server.ts index 44860e526..01a5b4bda 100644 --- a/services/discordbot/src/server.ts +++ b/services/discordbot/src/server.ts @@ -40,6 +40,9 @@ const options: DiscordbotOptions = { publicKey, discordApiUrl: optionalEnv("DISCORD_API_URL"), guildAllowlist: optionalList("DISCORDBOT_GUILD_ALLOWLIST"), + // Channels where "@bot status" answers with the control-plane report; + // unset = the status fast-path is disabled. + statusChannelAllowlist: optionalList("DISCORDBOT_STATUS_CHANNEL_IDS"), idleTimeoutMs: optionalNumberEnv("SESSION_IDLE_TIMEOUT_MS"), isGatewayActive: () => gateway.isActive(), maxConcurrentExecutionsPerGuild: optionalNumberEnv( diff --git a/services/discordbot/src/status.ts b/services/discordbot/src/status.ts index 506f4e5cb..789d86715 100644 --- a/services/discordbot/src/status.ts +++ b/services/discordbot/src/status.ts @@ -1,19 +1,13 @@ import type { DiscordbotFetch } from "./types"; -import { errorMessage, sliceSurrogateSafe } from "./utils"; +import { sliceSurrogateSafe } from "./utils"; // A "status" mention answers directly from the control plane — no sandbox, no -// session turn. The whole point is that it still works when the agent pipeline -// is broken: api-rs health comes from its /healthz + /readyz endpoints, and the -// turn/sandbox history comes from the shared session database (the same -// Postgres api-rs writes session_executions/sessions/session_warm_sandboxes -// to, reached via the bot's DATABASE_URL). Every source is fetched -// independently and best-effort so one dead dependency never blanks the rest -// of the report. - -/** Structural slice of pg.Pool so tests can stub the database trivially. */ -export type StatusDb = { - query(sql: string): Promise<{ rows: Record[] }>; -}; +// session turn — so it still works when the agent pipeline is what's broken. +// All data comes from api-rs over HTTP: /healthz + /readyz for liveness, and +// the read-only /api/status report (api-rs owns the session schema; this +// service deliberately runs no SQL). Every fetch carries a timeout and is +// best-effort, so one dead dependency never blanks the rest of the report or +// hangs the per-thread handler lock. const KEYWORD = /^(status|health)[?!.]*$/i; // Raw Discord mention markup (<@123>, <@!123>, <@&role>, <#channel>) plus the @@ -21,7 +15,7 @@ const KEYWORD = /^(status|health)[?!.]*$/i; const MENTION_TOKEN = /^(<[@#][!&]?\w+>|@[\w.-]+)$/; /** - * True when the message is ONLY a status request ("@gerard status", + * True when the message is ONLY a status request ("@bot status", * "<@&123> health?"). Anything with more words ("status of the deploy") falls * through to a normal agent turn so real questions are never hijacked. */ @@ -52,32 +46,31 @@ export type DailyRow = { }; export type StatusReport = { + /** null = unreachable, false = responded unhealthy, true = healthy. */ apiHealthy: boolean | null; apiReady: boolean | null; - collectedNotes: string[]; - /** Runs per UTC day, oldest→today, zero-filled to exactly 7 entries. */ daily: DailyRow[]; - dbOk: boolean; inFlight: ExecutionRow[]; recent: ExecutionRow[]; - sandboxes: { ageSeconds: number | null; sandboxId: string; threadKey: string }[]; + /** Whether the /api/status report fetch succeeded. */ + reportOk: boolean; + sandboxes: { idleSeconds: number | null; sandboxId: string; threadKey: string }[]; tally: Record; warmPool: Record; }; const HEALTH_TIMEOUT_MS = 2_000; -const ERROR_SNIPPET_CHARS = 150; +const REPORT_TIMEOUT_MS = 5_000; export async function collectStatus(input: { apiUrl: string; - db: StatusDb | null; fetchFn?: DiscordbotFetch; nowMs?: number; }): Promise { const fetchFn = input.fetchFn ?? fetch; const now = input.nowMs ?? Date.now(); - const notes: string[] = []; + // null = unreachable (nothing answered), false = answered non-2xx. const probe = async (path: string): Promise => { try { const response = await fetchFn(`${input.apiUrl}${path}`, { @@ -85,120 +78,65 @@ export async function collectStatus(input: { }); return response.ok; } catch { - return false; + return null; } }; - const query = async ( - label: string, - sql: string, - ): Promise[] | null> => { - if (!input.db) return null; + const fetchReport = async (): Promise | null> => { try { - return (await input.db.query(sql)).rows; - } catch (error) { - notes.push(`${label} unavailable (${errorMessage(error)})`); + const response = await fetchFn(`${input.apiUrl}/api/status`, { + signal: AbortSignal.timeout(REPORT_TIMEOUT_MS), + }); + if (!response.ok) return null; + const body: unknown = await response.json(); + return typeof body === "object" && body !== null + ? (body as Record) + : null; + } catch { return null; } }; + const [apiHealthy, apiReady, report] = await Promise.all([ + probe("/healthz"), + probe("/readyz"), + fetchReport(), + ]); + + const rows = (key: string): Record[] => { + const value = report?.[key]; + return Array.isArray(value) + ? value.filter( + (row): row is Record => + typeof row === "object" && row !== null, + ) + : []; + }; + const toExecutionRow = (row: Record): ExecutionRow => ({ - ageSeconds: ageSecondsFrom(row.created_at, now), + ageSeconds: numberOrNull(row.age_seconds), durationSeconds: numberOrNull(row.duration_seconds), - error: String(row.error ?? "").slice(0, ERROR_SNIPPET_CHARS), + error: String(row.error ?? ""), status: String(row.status ?? "unknown"), threadKey: String(row.thread_key ?? "?"), title: String(row.title ?? ""), who: String(row.user_name ?? ""), }); - const [apiHealthy, apiReady, recent, tally, inFlight, sandboxes, warm, daily] = - await Promise.all([ - probe("/healthz"), - probe("/readyz"), - query( - "recent turns", - `SELECT e.thread_key, e.status, - left(coalesce(e.error, ''), ${ERROR_SNIPPET_CHARS}) AS error, - e.created_at, - extract(epoch FROM (e.completed_at - e.started_at)) AS duration_seconds, - e.metadata ->> 'user_name' AS user_name, - coalesce(s.title, s.metadata ->> 'discord_conversation_name', - s.metadata ->> 'linear_conversation_name', - s.metadata ->> 'slack_conversation_name') AS title - FROM session_executions e - LEFT JOIN sessions s ON s.thread_key = e.thread_key - ORDER BY e.created_at DESC - LIMIT 8`, - ), - query( - "24h tally", - `SELECT status, count(*)::int AS count - FROM session_executions - WHERE created_at > now() - interval '24 hours' - GROUP BY status`, - ), - query( - "in-flight turns", - `SELECT e.thread_key, e.status, '' AS error, e.created_at, - NULL AS duration_seconds, - e.metadata ->> 'user_name' AS user_name, - coalesce(s.title, s.metadata ->> 'discord_conversation_name', - s.metadata ->> 'linear_conversation_name', - s.metadata ->> 'slack_conversation_name') AS title - FROM session_executions e - LEFT JOIN sessions s ON s.thread_key = e.thread_key - WHERE e.status IN ('queued', 'running') - ORDER BY e.created_at ASC - LIMIT 8`, - ), - query( - "active sandboxes", - `SELECT thread_key, sandbox_id, sandbox_last_active_at - FROM sessions - WHERE sandbox_id IS NOT NULL - AND sandbox_last_active_at > now() - interval '2 hours' - ORDER BY sandbox_last_active_at DESC - LIMIT 8`, - ), - // Claimed/failed rows are never deleted — they're lifetime history, so - // an unfiltered count reads like a leak ("868 claimed"). Only ready/ - // evicting are current facts; show claimed/failed as 24h churn. - query( - "warm pool", - `SELECT status, count(*)::int AS count - FROM session_warm_sandboxes - WHERE status IN ('ready', 'evicting') - OR updated_at > now() - interval '24 hours' - GROUP BY status`, - ), - query( - "7-day histogram", - `SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS day, - count(*)::int AS runs, - (count(*) FILTER (WHERE status = 'failed'))::int AS failed - FROM session_executions - WHERE created_at > now() - interval '7 days' - GROUP BY 1 - ORDER BY 1`, - ), - ]); - return { apiHealthy, apiReady, - collectedNotes: notes, - daily: zeroFilledWeek(daily ?? [], now), - dbOk: recent !== null, - inFlight: (inFlight ?? []).map(toExecutionRow), - recent: (recent ?? []).map(toExecutionRow), - sandboxes: (sandboxes ?? []).map((row) => ({ - ageSeconds: ageSecondsFrom(row.sandbox_last_active_at, now), + daily: zeroFilledWeek(rows("daily"), now), + inFlight: rows("in_flight").map(toExecutionRow), + recent: rows("recent_executions").map(toExecutionRow), + reportOk: report !== null, + sandboxes: rows("active_sandboxes").map((row) => ({ + idleSeconds: numberOrNull(row.idle_seconds), sandboxId: String(row.sandbox_id ?? "?"), threadKey: String(row.thread_key ?? "?"), })), - tally: countsByStatus(tally), - warmPool: countsByStatus(warm), + tally: countsByStatus(rows("tally_24h")), + warmPool: countsByStatus(rows("warm_pool")), }; } @@ -218,15 +156,11 @@ const STATUS_TAG: Record = { const TAG_WIDTH = 5; const THREAD_WIDTH = 24; const WHO_WIDTH = 10; - -// Internal actor ids nobody recognizes → the name the team knows. -const WHO_ALIAS: Record = { - "github-pr-manager": "gerard", -}; const AGE_WIDTH = 4; const DUR_WIDTH = 5; const ERROR_LINE_CHARS = 60; const BAR_WIDTH = 16; +const RECENT_ROWS_SHOWN = 8; const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; @@ -239,14 +173,16 @@ function weekdayLabel(dayIso: string): string { /** * Discord has no table markup; the closest thing is a monospace code block * with hand-padded columns. Header line stays OUTSIDE the block (bold + emoji - * work there); rows stay ~50 chars wide to limit wrapping on mobile. + * work there); rows stay ~50 chars wide to limit wrapping on mobile. The + * histogram gets its own second block. `botName` labels the header — this is + * generic service code, so the deployment's bot name is a parameter. */ -export function formatStatus(report: StatusReport): string { +export function formatStatus(report: StatusReport, botName: string): string { const mark = (value: boolean | null): string => value === null ? "❓" : value ? "✅" : "❌"; const header = - `**gerard status** · api-rs ${mark(report.apiHealthy)} ` + - `ready ${mark(report.apiReady)} · db ${report.dbOk ? "✅" : "❌"}`; + `**${botName} status** · api-rs ${mark(report.apiHealthy)} ` + + `ready ${mark(report.apiReady)} · data ${report.reportOk ? "✅" : "❌"}`; const lines: string[] = []; @@ -272,27 +208,27 @@ export function formatStatus(report: StatusReport): string { `${took.padStart(DUR_WIDTH)}`; // One table: in-flight turns first (no duration yet), then settled recent - // turns. The recent query also returns queued/running rows — skip those so + // turns. The recent list also carries queued/running rows — skip those so // an in-flight turn isn't listed twice. AGE = when the turn was requested, // TOOK = how long it ran. const turnRow = (row: ExecutionRow): string => tableRow( STATUS_TAG[row.status] ?? row.status, - threadLabel(row), - WHO_ALIAS[row.who] ?? row.who, + inline(threadLabel(row)), + inline(row.who), formatAge(row.ageSeconds), row.durationSeconds !== null ? formatDuration(row.durationSeconds) : "-", ).trimEnd(); - const settled = report.recent.filter( - (row) => row.status !== "queued" && row.status !== "running", - ); + const settled = report.recent + .filter((row) => row.status !== "queued" && row.status !== "running") + .slice(0, RECENT_ROWS_SHOWN); const turns = [...report.inFlight, ...settled]; if (turns.length > 0) { lines.push(tableRow("", "THREAD", "WHO", "AGE", "TOOK").trimEnd()); for (const row of turns) { lines.push(turnRow(row)); if (row.error) { - lines.push(` └ ${row.error.slice(0, ERROR_LINE_CHARS)}`); + lines.push(` └ ${inline(row.error).slice(0, ERROR_LINE_CHARS)}`); } } } @@ -302,10 +238,10 @@ export function formatStatus(report: StatusReport): string { sandboxBits.push(`${report.sandboxes.length} active`); } // ready/evicting are the pool's current state; claimed/failed rows are - // historical (the collect query already windows them to 24h). "failed" here - // is a warm SPAWN failure (a standby sandbox that didn't provision — the - // next session cold-starts instead), NOT a failed turn; label it so it - // can't be confused with the histogram's FAIL column. + // historical (the report windows them to 24h). "failed" here is a warm + // SPAWN failure (a standby sandbox that didn't provision — the next session + // cold-starts instead), NOT a failed turn; label it so it can't be confused + // with the histogram's FAIL column. const WARM_LABEL: Record = { failed: "spawn-failed" }; const warmLine = (statuses: string[]): string => statuses @@ -323,9 +259,8 @@ export function formatStatus(report: StatusReport): string { lines.push(`sandboxes: ${sandboxBits.join(" · ")}`); } - for (const note of report.collectedNotes) lines.push(`! ${note}`); - if (!report.dbOk && report.collectedNotes.length === 0) { - lines.push("! session database unreachable — turn history unavailable"); + if (!report.reportOk) { + lines.push("! status report unavailable — turn history not shown"); } // 7-day histogram, in its OWN code block below the live view: the bar @@ -363,8 +298,7 @@ export function formatStatus(report: StatusReport): string { const body = lines.join("\n"); // The histogram block is small and fixed-size; give the live view whatever // budget remains under Discord's cap. - const budget = - STATUS_MAX_CHARS - header.length - histogramBlock.length - 20; + const budget = STATUS_MAX_CHARS - header.length - histogramBlock.length - 20; const bounded = body.length <= budget ? body @@ -373,6 +307,9 @@ export function formatStatus(report: StatusReport): string { return `${header}${liveBlock}${histogramBlock}`; } +/** Generic failure reply — internals go to logs, not the channel. */ +export const STATUS_FAILURE_REPLY = "⚠️ status check failed — see service logs."; + /** * Human label for a turn: the session title when the bots set one, otherwise * a friendlier rendering of the thread key ("GH PR splits-teams#1799" beats @@ -394,12 +331,26 @@ function threadLabel(row: { threadKey: string; title: string }): string { return row.threadKey; } +/** + * Neutralize markdown/code-fence breakouts in interpolated values (titles and + * error strings are user/agent-influenced): backticks become apostrophes and + * whitespace collapses to single spaces so a value can never close the + * surrounding fence or smuggle its own line. + */ +function inline(value: string): string { + return value.replace(/`/g, "'").replace(/\s+/g, " ").trim(); +} + /** * Truncate + pad to the column. Middle ellipsis by default so both ends stay * readable ("GH PR splits-con…eams#1799", "Discord 90294…:1391220231" — the * head names the thing, the tail discriminates); plain head-cut for names. */ -function fit(value: string, width: number, keep: "edges" | "head" = "edges"): string { +function fit( + value: string, + width: number, + keep: "edges" | "head" = "edges", +): string { if (value.length <= width) return value.padEnd(width); if (keep === "head" || width < 12) { return `${value.slice(0, width - 1)}…`; @@ -444,25 +395,16 @@ function zeroFilledWeek( } function countsByStatus( - rows: Record[] | null, + rows: Record[], ): Record { const counts: Record = {}; - for (const row of rows ?? []) { + for (const row of rows) { const count = numberOrNull(row.count); if (count !== null) counts[String(row.status ?? "unknown")] = count; } return counts; } -function ageSecondsFrom(value: unknown, nowMs: number): number | null { - if (value instanceof Date) return (nowMs - value.getTime()) / 1000; - if (typeof value === "string") { - const parsed = Date.parse(value); - if (!Number.isNaN(parsed)) return (nowMs - parsed) / 1000; - } - return null; -} - function numberOrNull(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim() !== "") { diff --git a/services/discordbot/src/types.ts b/services/discordbot/src/types.ts index 9e5df03ee..574fc9d65 100644 --- a/services/discordbot/src/types.ts +++ b/services/discordbot/src/types.ts @@ -114,6 +114,13 @@ export type DiscordbotOptions = { recoverRenderObligationsOnStart?: boolean; state?: StateAdapter; stateKeyPrefix?: string; + /** + * Channel (or thread) ids where a bare "status"/"health" mention gets the + * control-plane status reply. Empty/unset disables the fast-path entirely — + * the report surfaces cross-platform activity (session titles, requester + * names, error snippets), so exposure is an explicit per-channel opt-in. + */ + statusChannelAllowlist?: readonly string[]; /** * Discord delta (mirrors slackbotv2's `triggerBotAllowlist`): bot user ids * whose messages may trigger/append despite being bot-authored. diff --git a/services/discordbot/test/status.test.ts b/services/discordbot/test/status.test.ts index 86441cc20..1754965c8 100644 --- a/services/discordbot/test/status.test.ts +++ b/services/discordbot/test/status.test.ts @@ -3,7 +3,6 @@ import { collectStatus, formatStatus, isStatusCommand, - type StatusDb, type StatusReport, } from "../src/status"; import type { DiscordbotFetch } from "../src/types"; @@ -30,107 +29,102 @@ describe("isStatusCommand", () => { }); }); -function healthyFetch(status = 200): DiscordbotFetch { - return async () => new Response("ok", { status }); -} +const NOW = Date.parse("2026-08-12T12:00:00Z"); -function stubDb(handler: (sql: string) => Record[]): StatusDb { - return { - async query(sql: string) { - return { rows: handler(sql) }; +const FULL_REPORT = { + ok: true, + recent_executions: [ + { + age_seconds: 300, + duration_seconds: 63, + error: null, + status: "completed", + thread_key: "github-manage:0xSplits/splits-teams:1799", + title: null, + user_name: "0xdiid", }, - }; -} - -const NOW = Date.parse("2026-08-12T12:00:00Z"); + { + age_seconds: 1900, + duration_seconds: 12, + error: "sandbox spawn timeout after 120s", + status: "failed", + thread_key: "discord:1:2:9", + title: null, + user_name: "jaan", + }, + ], + in_flight: [ + { + age_seconds: 120, + duration_seconds: null, + error: null, + status: "running", + thread_key: "discord:1:2:3", + title: "fix the deploy pipeline", + user_name: "oliver", + }, + ], + tally_24h: [ + { count: 41, status: "completed" }, + { count: 2, status: "failed" }, + ], + active_sandboxes: [ + { + idle_seconds: 60, + sandbox_id: "asbx-1755000000-1", + thread_key: "discord:1:2:3", + }, + ], + warm_pool: [ + { count: 2, status: "ready" }, + { count: 41, status: "claimed" }, + ], + daily: [ + { day: "2026-08-10", failed: 0, runs: 12 }, + { day: "2026-08-11", failed: 3, runs: 40 }, + { day: "2026-08-12", failed: 0, runs: 20 }, + ], +}; -function fullDb(): StatusDb { - return stubDb((sql) => { - if (sql.includes("session_warm_sandboxes")) { - return [ - { status: "ready", count: 2 }, - { status: "claimed", count: 41 }, - ]; - } - if (sql.includes("interval '7 days'")) { - return [ - { day: "2026-08-10", failed: 0, runs: 12 }, - { day: "2026-08-11", failed: 3, runs: 40 }, - { day: "2026-08-12", failed: 0, runs: 20 }, - ]; - } - if (sql.includes("interval '24 hours'")) { - return [ - { status: "completed", count: 41 }, - { status: "failed", count: 2 }, - ]; - } - if (sql.includes("IN ('queued', 'running')")) { - return [ - { - created_at: new Date(NOW - 120_000), - duration_seconds: null, - error: "", - status: "running", - thread_key: "discord:1:2:3", - title: "fix the deploy pipeline", - user_name: "oliver", - }, - ]; +function apiFetch(input: { + health?: number; + ready?: number; + report?: unknown; + reportStatus?: number; +}): DiscordbotFetch { + return async (url) => { + const path = String(url); + if (path.endsWith("/healthz")) { + return new Response("{}", { status: input.health ?? 200 }); } - if (sql.includes("FROM session_executions")) { - return [ - { - created_at: new Date(NOW - 300_000), - duration_seconds: "63", - error: "", - status: "completed", - thread_key: "github-manage:0xSplits/splits-teams:1799", - title: null, - user_name: "0xdiid", - }, - { - created_at: new Date(NOW - 1_900_000), - duration_seconds: "12", - error: "sandbox spawn timeout after 120s", - status: "failed", - thread_key: "discord:1:2:9", - title: null, - user_name: "jaan", - }, - ]; + if (path.endsWith("/readyz")) { + return new Response("{}", { status: input.ready ?? 200 }); } - if (sql.includes("FROM sessions")) { - return [ - { - sandbox_id: "asbx-1755000000-1", - sandbox_last_active_at: new Date(NOW - 60_000), - thread_key: "discord:1:2:3", - }, - ]; + if (path.endsWith("/api/status")) { + return new Response(JSON.stringify(input.report ?? FULL_REPORT), { + status: input.reportStatus ?? 200, + }); } - return []; - }); + throw new Error(`unexpected fetch: ${path}`); + }; } describe("collectStatus", () => { it("assembles a full report when everything is up", async () => { const report = await collectStatus({ apiUrl: "http://api", - db: fullDb(), - fetchFn: healthyFetch(), + fetchFn: apiFetch({}), nowMs: NOW, }); expect(report.apiHealthy).toBe(true); expect(report.apiReady).toBe(true); - expect(report.dbOk).toBe(true); + expect(report.reportOk).toBe(true); expect(report.tally).toEqual({ completed: 41, failed: 2 }); expect(report.recent).toHaveLength(2); expect(report.recent[1]?.error).toContain("sandbox spawn timeout"); expect(report.inFlight).toHaveLength(1); expect(report.sandboxes[0]?.sandboxId).toBe("asbx-1755000000-1"); expect(report.warmPool).toEqual({ claimed: 41, ready: 2 }); - expect(report.collectedNotes).toEqual([]); // Zero-filled to exactly 7 UTC days, oldest first, today last. expect(report.daily).toHaveLength(7); expect(report.daily[0]).toEqual({ day: "2026-08-06", failed: 0, runs: 0 }); @@ -138,44 +132,53 @@ describe("collectStatus", () => { expect(report.daily[6]).toEqual({ day: "2026-08-12", failed: 0, runs: 20 }); }); - it("still reports DB data when api-rs is down", async () => { + it("marks api-rs unreachable (null) but still parses the report", async () => { + const fetchFn: DiscordbotFetch = async (url) => { + const path = String(url); + if (path.endsWith("/healthz") || path.endsWith("/readyz")) { + throw new Error("connect ECONNREFUSED"); + } + return new Response(JSON.stringify(FULL_REPORT), { status: 200 }); + }; const report = await collectStatus({ apiUrl: "http://api", - db: fullDb(), - fetchFn: async () => { - throw new Error("connect ECONNREFUSED"); - }, + fetchFn, + nowMs: NOW, + }); + expect(report.apiHealthy).toBeNull(); + expect(report.apiReady).toBeNull(); + expect(report.reportOk).toBe(true); + }); + + it("distinguishes unhealthy (false) from unreachable (null)", async () => { + const report = await collectStatus({ + apiUrl: "http://api", + fetchFn: apiFetch({ ready: 503 }), nowMs: NOW, }); - expect(report.apiHealthy).toBe(false); + expect(report.apiHealthy).toBe(true); expect(report.apiReady).toBe(false); - expect(report.dbOk).toBe(true); - expect(report.recent).toHaveLength(2); }); - it("still reports api-rs health when the DB is down", async () => { + it("still reports health when the status report fails", async () => { const report = await collectStatus({ apiUrl: "http://api", - db: stubDb(() => { - throw new Error("password authentication failed"); - }), - fetchFn: healthyFetch(), + fetchFn: apiFetch({ reportStatus: 500 }), nowMs: NOW, }); expect(report.apiHealthy).toBe(true); - expect(report.dbOk).toBe(false); - expect(report.collectedNotes.length).toBeGreaterThan(0); + expect(report.reportOk).toBe(false); expect(report.recent).toEqual([]); + expect(report.daily.every((day) => day.runs === 0)).toBe(true); }); - it("handles a missing database configuration", async () => { + it("tolerates a malformed report body", async () => { const report = await collectStatus({ apiUrl: "http://api", - db: null, - fetchFn: healthyFetch(), + fetchFn: apiFetch({ report: "not an object" }), nowMs: NOW, }); - expect(report.dbOk).toBe(false); + expect(report.reportOk).toBe(false); expect(report.recent).toEqual([]); }); }); @@ -184,11 +187,10 @@ describe("formatStatus", () => { const baseReport = (): StatusReport => ({ apiHealthy: true, apiReady: true, - collectedNotes: [], daily: [], - dbOk: true, inFlight: [], recent: [], + reportOk: true, sandboxes: [], tally: {}, warmPool: {}, @@ -197,29 +199,15 @@ describe("formatStatus", () => { it("renders a code-block table with tags, tallies, and sandboxes", async () => { const report = await collectStatus({ apiUrl: "http://api", - db: fullDb(), - fetchFn: healthyFetch(), + fetchFn: apiFetch({}), nowMs: NOW, }); - const text = formatStatus(report); - // Header outside the block, data inside one fenced block. - expect(text.startsWith("**gerard status** · api-rs ✅")).toBe(true); - expect(text).toContain("```"); + const text = formatStatus(report, "centaur"); + // Header names the bot and stays outside the block. + expect(text.startsWith("**centaur status** · api-rs ✅")).toBe(true); expect(text).toContain("24h: 41 ok · 2 FAIL"); // Column headings above the turn table. expect(text).toMatch(/THREAD\s+WHO\s+AGE\s+TOOK/); - // 7-day histogram in its OWN code block, after the live view: full-width - // bar on the busiest day, "-" for zero failures, zero-run days barless, - // and a failure-rate stat line. - expect(text.split("```")).toHaveLength(5); - expect(text.indexOf("LAST 7 DAYS")).toBeGreaterThan( - text.indexOf("sandboxes:"), - ); - expect(text).toMatch(/LAST 7 DAYS\s+RUNS FAIL/); - expect(text).toMatch(/Tue {2}█{16}\s+40\s+3/); - expect(text).toMatch(/Wed {2}█+\s+20\s+-/); - expect(text).toMatch(/Thu {2}\s+0\s+-/); - expect(text).toContain("7d: 72 runs · 3 failed (4.2%)"); // In-flight row first: session title, requester, no duration yet. const lines = text.split("\n"); const runLine = lines.find((line) => line.startsWith("run")); @@ -234,9 +222,38 @@ describe("formatStatus", () => { expect(text).toContain( "sandboxes: 1 active · warm: 2 ready · warm 24h: 41 claimed", ); + // Histogram in its OWN code block, after the live view. + expect(text.split("```")).toHaveLength(5); + expect(text.indexOf("LAST 7 DAYS")).toBeGreaterThan( + text.indexOf("sandboxes:"), + ); + expect(text).toMatch(/Tue {2}█{16}\s+40\s+3/); + expect(text).toMatch(/Wed {2}█+\s+20\s+-/); + expect(text).toMatch(/Thu {2}\s+0\s+-/); + expect(text).toContain("7d: 72 runs · 3 failed (4.2%)"); expect(text.length).toBeLessThanOrEqual(2000); }); + it("neutralizes backticks and newlines in titles and errors", () => { + const report = baseReport(); + report.recent = [ + { + ageSeconds: 60, + durationSeconds: 5, + error: "boom ``` **bold**\nnext line", + status: "failed", + threadKey: "discord:1:2", + title: "evil ``` title", + who: "someone", + }, + ]; + const text = formatStatus(report, "centaur"); + // Exactly the wrapper's own fence pair — no fences leaked from values. + expect(text.split("```")).toHaveLength(3); + expect(text).toContain("evil ''' title"); + expect(text).toContain("boom ''' **bold** next line"); + }); + it("keeps thread rows within the column budget", () => { const report = baseReport(); report.recent = [ @@ -250,7 +267,7 @@ describe("formatStatus", () => { who: "someone-with-a-long-name", }, ]; - const text = formatStatus(report); + const text = formatStatus(report, "centaur"); const row = text.split("\n").find((line) => line.startsWith("ok")); expect(row).toBeDefined(); // Middle ellipsis keeps the platform head and the id tail. @@ -260,15 +277,16 @@ describe("formatStatus", () => { expect(row?.length ?? 0).toBeLessThanOrEqual(52); }); - it("marks a down api-rs and unreachable DB honestly", () => { + it("marks a down api-rs and missing report honestly", () => { const report = baseReport(); - report.apiHealthy = false; + report.apiHealthy = null; report.apiReady = false; - report.dbOk = false; - const text = formatStatus(report); - expect(text).toContain("api-rs ❌"); - expect(text).toContain("db ❌"); - expect(text).toContain("session database unreachable"); + report.reportOk = false; + const text = formatStatus(report, "centaur"); + expect(text).toContain("api-rs ❓"); + expect(text).toContain("ready ❌"); + expect(text).toContain("data ❌"); + expect(text).toContain("status report unavailable"); }); it("stays under the Discord cap with oversized errors", () => { @@ -282,9 +300,8 @@ describe("formatStatus", () => { title: "", who: "someone", })); - const text = formatStatus(report); + const text = formatStatus(report, "centaur"); expect(text.length).toBeLessThanOrEqual(2000); expect(text.endsWith("```")).toBe(true); - expect(text).toContain("[truncated]"); }); }); From aee3bc7393c4e62ae6a3e4828d866beba3a75204 Mon Sep 17 00:00:00 2001 From: Oliver Ponder Date: Wed, 12 Aug 2026 21:55:05 +0200 Subject: [PATCH 10/10] feat(api-rs): read-only /api/status operational report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /api/status returns a JSON snapshot for status surfaces (chat-bot status commands, dashboards): recent and in-flight executions (with session title, requester, truncated error), a 24h status tally, active session sandboxes, warm-pool state (current ready/evicting plus 24h claimed/failed churn — those rows are lifetime history), and a 7-calendar- day UTC run histogram. api-rs owns the session schema, so the SQL lives here instead of in every ingress service that wants a status view. The report is cached in-process for 10s so scripted callers cost at most one scan set per TTL, and a new migration indexes session_executions(created_at) for the global recency scans. Co-Authored-By: Claude Fable 5 --- .../crates/centaur-api-server/src/lib.rs | 1 + .../crates/centaur-api-server/src/routes.rs | 3 +- .../crates/centaur-api-server/src/status.rs | 183 ++++++++++++++++++ ...0049_session_executions_created_at_idx.sql | 7 + 4 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 services/api-rs/crates/centaur-api-server/src/status.rs create mode 100644 services/api-rs/crates/centaur-session-sqlx/migrations/0049_session_executions_created_at_idx.sql diff --git a/services/api-rs/crates/centaur-api-server/src/lib.rs b/services/api-rs/crates/centaur-api-server/src/lib.rs index 6162375c2..36945b41c 100644 --- a/services/api-rs/crates/centaur-api-server/src/lib.rs +++ b/services/api-rs/crates/centaur-api-server/src/lib.rs @@ -4,6 +4,7 @@ mod error; mod mcp; mod routes; mod slack_proxy; +mod status; mod tool_discovery; pub mod types; diff --git a/services/api-rs/crates/centaur-api-server/src/routes.rs b/services/api-rs/crates/centaur-api-server/src/routes.rs index 83a0e23e9..4c1a981ad 100644 --- a/services/api-rs/crates/centaur-api-server/src/routes.rs +++ b/services/api-rs/crates/centaur-api-server/src/routes.rs @@ -160,7 +160,7 @@ impl AppState { .ok_or_else(|| ApiError::BadRequest("workflow runtime is not enabled".to_owned())) } - fn pool(&self) -> Result { + pub(crate) fn pool(&self) -> Result { let initialized = self .initialized() .ok_or_else(|| ApiError::ServiceUnavailable("api-rs is still starting".to_owned()))?; @@ -209,6 +209,7 @@ pub fn build_router_with_app_state(state: AppState) -> Router { .route("/readyz", get(readyz)) .route("/metrics", get(metrics)) .route("/api/personas", get(list_personas)) + .route("/api/status", get(crate::status::status_report)) .route("/mcp", post(mcp_post).get(mcp_get)) .route( "/.well-known/oauth-protected-resource", diff --git a/services/api-rs/crates/centaur-api-server/src/status.rs b/services/api-rs/crates/centaur-api-server/src/status.rs new file mode 100644 index 000000000..ce1f726c0 --- /dev/null +++ b/services/api-rs/crates/centaur-api-server/src/status.rs @@ -0,0 +1,183 @@ +use std::{ + sync::{Mutex, OnceLock}, + time::{Duration, Instant}, +}; + +use axum::{Json, extract::State}; +use serde::Serialize; +use serde_json::{Value, json}; +use sqlx::PgPool; + +use crate::{error::ApiError, routes::AppState}; + +// Read-only operational snapshot for status surfaces (chat-bot "status" +// commands, dashboards): recent/in-flight executions, a 24h tally, active +// session sandboxes, warm-pool state, and a 7-calendar-day (UTC) run +// histogram. api-rs owns the session schema, so the SQL lives here rather +// than in every ingress service that wants a status view. +// +// The report is cached briefly in-process: the history queries scan +// session_executions (append-only, never pruned), and status commands are +// human-triggered but scriptable — the cache caps the database cost at one +// scan set per TTL regardless of how often callers ask. + +const CACHE_TTL: Duration = Duration::from_secs(10); +const ERROR_SNIPPET_CHARS: i32 = 200; +const RECENT_LIMIT: i64 = 20; + +static CACHE: OnceLock>> = OnceLock::new(); + +pub(crate) async fn status_report(State(state): State) -> Result, ApiError> { + let cache = CACHE.get_or_init(|| Mutex::new(None)); + if let Some((stored_at, report)) = cache + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() + && stored_at.elapsed() < CACHE_TTL + { + return Ok(Json(report)); + } + + let pool = state.pool()?; + let report = build_report(&pool).await?; + *cache + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((Instant::now(), report.clone())); + Ok(Json(report)) +} + +#[derive(Serialize, sqlx::FromRow)] +struct ExecutionRow { + /// Seconds since the execution was created. + age_seconds: f64, + duration_seconds: Option, + error: Option, + status: String, + thread_key: String, + title: Option, + user_name: Option, +} + +#[derive(Serialize, sqlx::FromRow)] +struct CountRow { + count: i64, + status: String, +} + +#[derive(Serialize, sqlx::FromRow)] +struct SandboxRow { + idle_seconds: f64, + sandbox_id: String, + thread_key: String, +} + +#[derive(Serialize, sqlx::FromRow)] +struct DailyRow { + /// UTC calendar date, `YYYY-MM-DD`. + day: String, + failed: i64, + runs: i64, +} + +async fn build_report(pool: &PgPool) -> Result { + let recent = sqlx::query_as::<_, ExecutionRow>( + "SELECT e.thread_key, e.status, \ + left(e.error, $1) AS error, \ + extract(epoch FROM (now() - e.created_at))::float8 AS age_seconds, \ + extract(epoch FROM (e.completed_at - e.started_at))::float8 AS duration_seconds, \ + e.metadata ->> 'user_name' AS user_name, \ + coalesce(s.title, s.metadata ->> 'discord_conversation_name', \ + s.metadata ->> 'linear_conversation_name', \ + s.metadata ->> 'slack_conversation_name') AS title \ + FROM session_executions e \ + LEFT JOIN sessions s ON s.thread_key = e.thread_key \ + ORDER BY e.created_at DESC \ + LIMIT $2", + ) + .bind(ERROR_SNIPPET_CHARS) + .bind(RECENT_LIMIT) + .fetch_all(pool) + .await?; + + let in_flight = sqlx::query_as::<_, ExecutionRow>( + "SELECT e.thread_key, e.status, \ + NULL::text AS error, \ + extract(epoch FROM (now() - e.created_at))::float8 AS age_seconds, \ + NULL::float8 AS duration_seconds, \ + e.metadata ->> 'user_name' AS user_name, \ + coalesce(s.title, s.metadata ->> 'discord_conversation_name', \ + s.metadata ->> 'linear_conversation_name', \ + s.metadata ->> 'slack_conversation_name') AS title \ + FROM session_executions e \ + LEFT JOIN sessions s ON s.thread_key = e.thread_key \ + WHERE e.status IN ('queued', 'running') \ + ORDER BY e.created_at ASC \ + LIMIT $1", + ) + .bind(RECENT_LIMIT) + .fetch_all(pool) + .await?; + + let tally_24h = sqlx::query_as::<_, CountRow>( + "SELECT status, count(*) AS count \ + FROM session_executions \ + WHERE created_at > now() - interval '24 hours' \ + GROUP BY status", + ) + .fetch_all(pool) + .await?; + + let active_sandboxes = sqlx::query_as::<_, SandboxRow>( + "SELECT thread_key, sandbox_id, \ + extract(epoch FROM (now() - sandbox_last_active_at))::float8 AS idle_seconds \ + FROM sessions \ + WHERE sandbox_id IS NOT NULL \ + AND sandbox_last_active_at > now() - interval '2 hours' \ + ORDER BY sandbox_last_active_at DESC \ + LIMIT $1", + ) + .bind(RECENT_LIMIT) + .fetch_all(pool) + .await?; + + // ready/evicting are the pool's current state. claimed/failed rows are + // lifetime history (claiming flips status in place, rows are never + // deleted), so an unfiltered count reads like a leak — window them to 24h + // churn instead. + let warm_pool = sqlx::query_as::<_, CountRow>( + "SELECT status, count(*) AS count \ + FROM session_warm_sandboxes \ + WHERE status IN ('ready', 'evicting') \ + OR updated_at > now() - interval '24 hours' \ + GROUP BY status", + ) + .fetch_all(pool) + .await?; + + // Calendar-day (UTC) buckets over the last 7 days INCLUDING today, so the + // per-day rows and any total computed from them describe the same window + // (a rolling now()-7d fetch would include a partial 8th calendar day). + let daily = sqlx::query_as::<_, DailyRow>( + "SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD') AS day, \ + count(*) AS runs, \ + count(*) FILTER (WHERE status = 'failed') AS failed \ + FROM session_executions \ + WHERE created_at >= \ + (date_trunc('day', now() AT TIME ZONE 'UTC') - interval '6 days') \ + AT TIME ZONE 'UTC' \ + GROUP BY 1 \ + ORDER BY 1", + ) + .fetch_all(pool) + .await?; + + Ok(json!({ + "ok": true, + "active_sandboxes": active_sandboxes, + "daily": daily, + "in_flight": in_flight, + "recent_executions": recent, + "tally_24h": tally_24h, + "warm_pool": warm_pool, + })) +} diff --git a/services/api-rs/crates/centaur-session-sqlx/migrations/0049_session_executions_created_at_idx.sql b/services/api-rs/crates/centaur-session-sqlx/migrations/0049_session_executions_created_at_idx.sql new file mode 100644 index 000000000..4938c65fa --- /dev/null +++ b/services/api-rs/crates/centaur-session-sqlx/migrations/0049_session_executions_created_at_idx.sql @@ -0,0 +1,7 @@ +-- The /api/status report orders and windows session_executions by created_at +-- globally (ORDER BY created_at DESC LIMIT n; created_at > now() - '24 hours'; +-- the 7-day histogram). The existing (thread_key, created_at, execution_id) +-- index cannot serve a global recency scan over this append-only, never-pruned +-- table, so give created_at its own index. +create index if not exists session_executions_created_at_idx + on session_executions (created_at desc);