Make mention analysis replayable: persist extracted text, recompute on brand changes

.changeset/replayable-mention-analysis.md

@@ -0,0 +1,7 @@
+---
+"@workspace/web": patch
+"@workspace/worker": patch
+"@workspace/lib": patch
+---
+
+Changing a brand's name, aliases, domains, or competitors now recomputes mention data for past runs, and name matching no longer matches inside longer words.

apps/web/src/components/prompt-chart-print.tsx

@@ -28,7 +28,9 @@ interface PromptRunData {
 	version: string;
 	webSearchEnabled: boolean;
 	rawOutput: any;
+	textContent: string | null;
 	webQueries: string[];
+	analyzedAt: Date | null;
 }
 
 interface PromptChartPrintProps {

apps/web/src/lib/boss-client.ts

@@ -45,6 +45,12 @@ export async function getBoss(): Promise<PgBoss> {
 			retryBackoff: true,
 			expireInSeconds: 60 * 60,
 		});
+		await boss.createQueue("reanalyze-brand", {
+			retryLimit: 3,
+			retryDelay: 60,
+			retryBackoff: true,
+			expireInSeconds: 60 * 60,
+		});
 
 		bossInstance = boss;
 		return boss;

apps/web/src/lib/reanalysis.ts

@@ -0,0 +1,30 @@
+import { getBoss } from "@/lib/boss-client";
+
+/**
+ * Enqueue a re-analysis of all of a brand's historical prompt runs (and the
+ * branded/unbranded system tags of its prompts). Call after brand identity
+ * (name, aliases, website, domains) or competitors change so historical
+ * mention data reflects the new settings.
+ *
+ * Fire-and-forget: enqueue failures are logged but never block the settings
+ * save — the data just stays stale until the next change.
+ */
+export async function enqueueBrandReanalysis(brandId: string): Promise<void> {
+	try {
+		const boss = await getBoss();
+		await boss.send(
+			"reanalyze-brand",
+			{ brandId },
+			{
+				singletonKey: `reanalyze-${brandId}`,
+				singletonSeconds: 60, // debounce rapid consecutive settings edits
+				retryLimit: 3,
+				retryDelay: 60,
+				retryBackoff: true,
+				expireInSeconds: 60 * 60,
+			},
+		);
+	} catch (error) {
+		console.error(`Failed to enqueue re-analysis for brand ${brandId}:`, error);
+	}
+}

apps/web/src/routes/_authed/app/$brand/prompts/$promptId.tsx

@@ -660,7 +660,8 @@ function ResponsesTab({
 						<div>
 							<span className="text-xs text-muted-foreground block mb-1.5">LLM Response</span>
 							<div className="rounded-md border bg-muted/30 p-4 max-h-64 overflow-auto prose prose-sm max-w-none">
-								<ReactMarkdown>{extractTextContent(run.rawOutput, run.provider ?? run.model)}</ReactMarkdown>
+								{/* Prefer the persisted text; fall back to re-parsing rawOutput for rows the backfill hasn't reached. */}
+								<ReactMarkdown>{run.textContent ?? extractTextContent(run.rawOutput, run.provider ?? run.model)}</ReactMarkdown>
 							</div>
 						</div>
 

apps/web/src/routes/_authed/reports/render/$reportId.tsx

@@ -154,7 +154,8 @@ function ReportRenderPage() {
 				id: `run-${pi}-${ri}`, promptId, brandMentioned: run.brandMentioned,
 				competitorsMentioned: run.competitorsMentioned, createdAt: new Date(),
 				model: run.model, version: run.version,
-				webSearchEnabled: run.webSearchEnabled, rawOutput: run.rawOutput, webQueries: run.webQueries,
+				webSearchEnabled: run.webSearchEnabled, rawOutput: run.rawOutput,
+				textContent: run.textContent ?? null, webQueries: run.webQueries, analyzedAt: null,
 			});
 		});
 	});

apps/web/src/server/brands.ts

@@ -11,6 +11,7 @@ import { eq, and, count, sql } from "drizzle-orm";
 import { MAX_COMPETITORS } from "@workspace/lib/constants";
 import { cleanAndValidateDomain } from "@/lib/domain-categories";
 import { validateWebsiteUrl } from "@/lib/brand-website";
+import { enqueueBrandReanalysis } from "@/lib/reanalysis";
 import { parseScrapeTargets, selectTargetsForBrand } from "@workspace/lib/providers";
 import type { ModelConfig } from "@workspace/lib/providers";
 
@@ -239,6 +240,11 @@ export const updateBrandFn = createServerFn({ method: "POST" })
 			throw new Error("Failed to update brand");
 		}
 
+		// Brand identity changed - recompute historical mentions/system tags.
+		if (Object.keys(updateData).length > 0) {
+			await enqueueBrandReanalysis(data.brandId);
+		}
+
 		return result[0];
 	});
 
@@ -290,7 +296,7 @@ export const updateCompetitors = createServerFn({ method: "POST" })
 			};
 		});
 
-		return db.transaction(async (tx) => {
+		const saved = await db.transaction(async (tx) => {
 			await tx.delete(competitors).where(eq(competitors.brandId, data.brandId));
 
 			if (cleanedCompetitors.length > 0) {
@@ -308,6 +314,11 @@ export const updateCompetitors = createServerFn({ method: "POST" })
 				where: eq(competitors.brandId, data.brandId),
 			});
 		});
+
+		// Competitor list changed - recompute historical competitor mentions.
+		await enqueueBrandReanalysis(data.brandId);
+
+		return saved;
 	});
 
 /**
@@ -341,7 +352,10 @@ export const addDomainToBrandFn = createServerFn({ method: "POST" })
 			)
 			.returning();
 
-		if (result) return result;
+		if (result) {
+			await enqueueBrandReanalysis(data.brandId);
+			return result;
+		}
 
 		const brand = await db.query.brands.findFirst({
 			where: eq(brands.id, data.brandId),
@@ -381,6 +395,8 @@ export const addDomainToCompetitorFn = createServerFn({ method: "POST" })
 			.where(eq(competitors.id, data.competitorId))
 			.returning();
 
+		await enqueueBrandReanalysis(data.brandId);
+
 		return result;
 	});
 
@@ -420,5 +436,7 @@ export const createCompetitorFromDomainFn = createServerFn({ method: "POST" })
 			})
 			.returning();
 
+		await enqueueBrandReanalysis(data.brandId);
+
 		return result;
 	});

apps/worker/src/handlers.ts

@@ -4,6 +4,8 @@ import { processPromptJob, type ProcessPromptData } from "./jobs/process-prompt"
 import { generateReportJob, type GenerateReportData } from "./jobs/generate-report";
 import { scheduleMaintenanceJob, type ScheduleMaintenanceData } from "./jobs/schedule-maintenance";
 import { syncAuth0MembershipsJob, type SyncAuth0MembershipsData } from "./jobs/sync-auth0-memberships";
+import { backfillTextContentJob, type BackfillTextContentData } from "./jobs/backfill-text-content";
+import { reanalyzeBrandJob, type ReanalyzeBrandData } from "./jobs/reanalyze-brand";
 
 /** Wraps a pg-boss handler to report errors to Sentry before re-throwing. */
 function withSentry<T>(
@@ -48,6 +50,20 @@ export async function registerHandlers(boss: PgBoss): Promise<void> {
 	);
 	console.log("Registered handler: schedule-maintenance");
 
+	await boss.work<BackfillTextContentData>(
+		"backfill-text-content",
+		{ localConcurrency: 1 },
+		withSentry("backfill-text-content", backfillTextContentJob),
+	);
+	console.log("Registered handler: backfill-text-content");
+
+	await boss.work<ReanalyzeBrandData>(
+		"reanalyze-brand",
+		{ localConcurrency: 1 },
+		withSentry("reanalyze-brand", reanalyzeBrandJob),
+	);
+	console.log("Registered handler: reanalyze-brand");
+
 	if (process.env.DEPLOYMENT_MODE === "whitelabel") {
 		await boss.work<SyncAuth0MembershipsData>(
 			"sync-auth0-memberships",

apps/worker/src/index.ts

@@ -51,6 +51,18 @@ async function main() {
 		retryBackoff: true,
 		expireInSeconds: 60 * 30, // 30 minute timeout
 	});
+	await boss.createQueue("backfill-text-content", {
+		retryLimit: 3,
+		retryDelay: 300,
+		retryBackoff: true,
+		expireInSeconds: 60 * 60, // resumable — a retry continues where it left off
+	});
+	await boss.createQueue("reanalyze-brand", {
+		retryLimit: 3,
+		retryDelay: 60,
+		retryBackoff: true,
+		expireInSeconds: 60 * 60,
+	});
 	if (process.env.DEPLOYMENT_MODE === "whitelabel") {
 		await boss.createQueue("sync-auth0-memberships", {
 			retryLimit: 3,
@@ -69,6 +81,24 @@ async function main() {
 	);
 	console.log("Scheduled maintenance job (every 5 minutes)");
 
+	// One-off backfill of prompt_runs.text_content for historical rows. The
+	// job is idempotent (only touches rows where text_content IS NULL) and
+	// no-ops once the backfill is complete, so enqueueing on every startup is
+	// safe; the singleton key prevents pile-ups across restarts.
+	await boss.send(
+		"backfill-text-content",
+		{ source: "startup" },
+		{
+			singletonKey: "backfill-text-content",
+			singletonSeconds: 60 * 60,
+			retryLimit: 3,
+			retryDelay: 300,
+			retryBackoff: true,
+			expireInSeconds: 60 * 60,
+		},
+	);
+	console.log("Enqueued text-content backfill (singleton)");
+
 	if (process.env.DEPLOYMENT_MODE === "whitelabel") {
 		await boss.schedule(
 			"sync-auth0-memberships",

apps/worker/src/jobs/backfill-text-content.ts

@@ -0,0 +1,69 @@
+import type { Job } from "pg-boss";
+import { db } from "@workspace/lib/db/db";
+import { promptRuns } from "@workspace/lib/db/schema";
+import { tryExtractTextContent } from "@workspace/lib/text-extraction";
+import { and, asc, eq, gt, isNull } from "drizzle-orm";
+
+export interface BackfillTextContentData {
+	source?: string; // For logging - "startup" or "manual"
+}
+
+const BATCH_SIZE = 100;
+
+/**
+ * One-off backfill: populate prompt_runs.text_content for historical rows by
+ * re-extracting the answer text from raw_output.
+ *
+ * Batched (keyset pagination on id), resumable (only touches rows where
+ * text_content IS NULL), and idempotent — safe to enqueue on every worker
+ * startup. Rows whose raw_output yields no extractable text are left NULL.
+ */
+export async function backfillTextContentJob(jobs: Job<BackfillTextContentData>[]): Promise<void> {
+	for (const job of jobs) {
+		const source = job.data?.source || "startup";
+		console.log(`[backfill-text-content] Starting backfill (source: ${source})`);
+
+		let processed = 0;
+		let populated = 0;
+		let unextractable = 0;
+		let lastId: string | null = null;
+
+		for (;;) {
+			const rows = await db
+				.select({
+					id: promptRuns.id,
+					rawOutput: promptRuns.rawOutput,
+					provider: promptRuns.provider,
+					model: promptRuns.model,
+				})
+				.from(promptRuns)
+				.where(
+					and(isNull(promptRuns.textContent), ...(lastId ? [gt(promptRuns.id, lastId)] : [])),
+				)
+				.orderBy(asc(promptRuns.id))
+				.limit(BATCH_SIZE);
+
+			if (rows.length === 0) break;
+
+			for (const row of rows) {
+				const text = tryExtractTextContent(row.rawOutput, row.provider ?? row.model);
+				if (text !== null) {
+					await db.update(promptRuns).set({ textContent: text }).where(eq(promptRuns.id, row.id));
+					populated++;
+				} else {
+					unextractable++;
+				}
+				processed++;
+			}
+
+			lastId = rows[rows.length - 1].id;
+			console.log(
+				`[backfill-text-content] Progress: processed=${processed} populated=${populated} unextractable=${unextractable}`,
+			);
+		}
+
+		console.log(
+			`[backfill-text-content] Done: processed=${processed} populated=${populated} unextractable=${unextractable}`,
+		);
+	}
+}