diff --git a/.changeset/cli-lab-commands.md b/.changeset/cli-lab-commands.md new file mode 100644 index 00000000..fb1039c1 --- /dev/null +++ b/.changeset/cli-lab-commands.md @@ -0,0 +1,5 @@ +--- +"@elmohq/cli": minor +--- + +Added `elmo lab` commands to generate tracking prompts, evaluate them across providers (citations, mentions, share-of-voice, and query fan-out), and plan AEO improvements as one-off runs. diff --git a/apps/cli/package.json b/apps/cli/package.json index 447c9655..f6faf16e 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -29,21 +29,33 @@ "type": "module", "scripts": { "build": "node scripts/build.mjs", - "check-types": "tsc -p tsconfig.json" + "check-types": "tsc -p tsconfig.json", + "test": "vitest run" }, "dependencies": { + "@ai-sdk/anthropic": "^3.0.81", + "@ai-sdk/openai": "^3.0.68", + "@anthropic-ai/sdk": "^0.102.0", + "@brightdata/sdk": "^1.1.0", "@clack/prompts": "^1.5.1", + "ai": "^6.0.197", "commander": "^15.0.0", + "dataforseo-client": "^2.0.25", "dotenv": "17.4.2", + "marked": "^14.1.4", + "olostep": "^1.1.0", "picocolors": "^1.1.1", "posthog-node": "^5.36.4", - "semver": "^7.8.2" + "semver": "^7.8.2", + "zod": "^4.4.3" }, "devDependencies": { "@types/node": "^25.9.2", "@types/semver": "^7.7.1", "@workspace/config": "workspace:*", + "@workspace/lib": "workspace:*", "rolldown": "^1.1.0", - "typescript": "^6.0.3" + "typescript": "^6.0.3", + "vitest": "^4.1.8" } } diff --git a/apps/cli/src/commands/brainstorm.ts b/apps/cli/src/commands/brainstorm.ts new file mode 100644 index 00000000..9f95f6a6 --- /dev/null +++ b/apps/cli/src/commands/brainstorm.ts @@ -0,0 +1,97 @@ +import { analyzeBrand } from "@workspace/lib/onboarding"; +import type { Command } from "commander"; +import { suggestionToBrandPack } from "../core/brand-pack.js"; +import { loadElmoEnv } from "../core/env.js"; +import { parseFormat, printCsv, type Row, writeJson, writeStructured } from "../core/output.js"; +import { applyResearchTarget } from "../core/targets.js"; +import { log, routeLibraryLogsToStderr } from "../core/ui.js"; +import { trackCliEvent } from "../telemetry.js"; + +interface BrainstormOptions { + count: string; + competitors: string; + model?: string; + output: string; + format: string; + stdout?: boolean; + dir?: string; +} + +const PROMPT_COLUMNS = ["n", "type", "prompt", "tags"]; +const COMPETITOR_COLUMNS = ["name", "domains", "aliases"]; + +export function registerBrainstorm(lab: Command): void { + lab + .command("brainstorm") + .description("generate AI tracking prompts + competitors for a website") + .argument("", "website or domain to analyze (e.g. nike.com)") + .option("-c, --count ", "number of prompts to generate", "30") + .option("--competitors ", "number of competitors to generate", "10") + .option("-m, --model ", "research provider, model:provider[:version] (direct API only)") + .option("-o, --output ", "directory to write artifacts to", ".") + .option("--format ", "structured output format", "csv") + .option("--stdout", "print to stdout only; do not write files") + .action(async (website: string, _opts: object, cmd: Command) => { + const options = cmd.optsWithGlobals(); + await runBrainstorm(website, options); + }); +} + +async function runBrainstorm(website: string, options: BrainstormOptions): Promise { + routeLibraryLogsToStderr(); + const format = parseFormat(options.format); + const maxPrompts = parseCount(options.count, "count"); + const maxCompetitors = parseCount(options.competitors, "competitors"); + + const loaded = await loadElmoEnv(options.dir); + applyResearchTarget(options.model); + + log.step(`Analyzing ${website} (up to ${maxPrompts} prompts, ${maxCompetitors} competitors)…`); + const suggestion = await analyzeBrand({ website, maxPrompts, maxCompetitors }); + const pack = suggestionToBrandPack(suggestion); + + const brandLower = pack.brandName.toLowerCase(); + const promptRows: Row[] = pack.prompts.map((p, i) => ({ + n: i + 1, + type: brandLower && p.prompt.toLowerCase().includes(brandLower) ? "branded" : "unbranded", + prompt: p.prompt, + tags: p.tags, + })); + const competitorRows: Row[] = pack.competitors.map((c) => ({ + name: c.name, + domains: c.domains, + aliases: c.aliases, + })); + + // Machine-readable summary always goes to stdout. + printCsv(promptRows, PROMPT_COLUMNS); + + if (!options.stdout) { + const dir = options.output; + await writeJson(dir, "brand.json", pack); + await writeStructured(dir, "prompts", promptRows, PROMPT_COLUMNS, format); + await writeStructured(dir, "competitors", competitorRows, COMPETITOR_COLUMNS, format); + log.success( + `Wrote brand.json, prompts.${format}, competitors.${format} to ${dir} (${promptRows.length} prompts, ${competitorRows.length} competitors).`, + ); + log.info(`Pipe it into eval: elmo lab eval --brand-file ${dir.replace(/\/$/, "")}/brand.json -m `); + } + + if (loaded.configDir) { + await trackCliEvent(loaded.configDir, "cli_lab_brainstorm", { + prompt_count: promptRows.length, + competitor_count: competitorRows.length, + has_model: Boolean(options.model), + to_stdout: Boolean(options.stdout), + format, + }); + } +} + +function parseCount(value: string, name: string): number { + const n = Number(value); + if (!Number.isInteger(n) || n < 0) { + throw new Error(`--${name} must be a non-negative integer (got "${value}")`); + } + return n; +} diff --git a/apps/cli/src/commands/eval.ts b/apps/cli/src/commands/eval.ts new file mode 100644 index 00000000..ca96c3f8 --- /dev/null +++ b/apps/cli/src/commands/eval.ts @@ -0,0 +1,612 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { RUNS_PER_PROMPT, WEB_QUERIES_UNAVAILABLE } from "@workspace/lib/constants"; +import { analyzeMentions, type MentionBrand, type MentionCompetitor } from "@workspace/lib/mentions"; +import type { OnboardingCompetitor } from "@workspace/lib/onboarding"; +import { + computeCompetitorSoVs, + computeOverallSoV, + computePromptSoV, + type FullPromptRun, + findContentGaps, + type ReportCompetitor, + type ReportPromptRun, +} from "@workspace/lib/report-metrics"; +import type { Command } from "commander"; +import { + type BrandPack, + readBrandPack, + toMentionBrand, + toMentionCompetitors, + toReportCompetitors, +} from "../core/brand-pack.js"; +import { loadElmoEnv } from "../core/env.js"; +import { + type DataFormat, + ensureDir, + pad, + parseFormat, + printStdout, + type Row, + slugify, + toCsv, + writeJson, + writeStructured, + writeText, +} from "../core/output.js"; +import { mapPool } from "../core/pool.js"; +import { + buildEvalReportHtml, + type EvalPromptResult, + type EvalReport, + type EvalRun, + type EvalTargetResult, +} from "../core/report-html.js"; +import { type ResolvedTarget, resolveTargets } from "../core/targets.js"; +import { log, routeLibraryLogsToStderr } from "../core/ui.js"; +import { trackCliEvent } from "../telemetry.js"; + +interface EvalOptions { + model?: string[]; + runs: string; + prompt?: string[]; + promptsFile?: string; + brandFile?: string; + brand?: string; + brandDomain?: string; + alias?: string[]; + competitor?: string[]; + output: string; + format: string; + stdout?: boolean; + concurrency: string; + dir?: string; +} + +function collect(value: string, previous: string[] = []): string[] { + return previous.concat([value]); +} + +export function registerEval(lab: Command): void { + lab + .command("eval") + .description("run prompts across providers → responses, citations, mentions, share-of-voice, fan-out") + .argument("[prompts...]", "prompts to evaluate (or use --prompt / --prompts-file / --brand-file / stdin)") + .option("-m, --model ", "model:provider[:version][:online] (repeatable; default: SCRAPE_TARGETS)", collect) + .option("-n, --runs ", "replications per prompt per target", String(RUNS_PER_PROMPT)) + .option("--prompt ", "a prompt to evaluate (repeatable)", collect) + .option("--prompts-file ", "file with one prompt per line ('-' for stdin)") + .option("--brand-file ", "brand pack JSON (from `elmo lab brainstorm`) for prompts + mention context") + .option("--brand ", "brand name for mention detection") + .option("--brand-domain ", "brand website/domain for mention detection") + .option("--alias ", "brand alias for mention detection (repeatable)", collect) + .option("--competitor ", "competitor for share-of-voice (repeatable)", collect) + .option("-o, --output ", "directory to write artifacts to", ".") + .option("--format ", "structured output format", "csv") + .option("--stdout", "print to stdout only; do not write files") + .option("--concurrency ", "max concurrent provider calls", "4") + .action(async (prompts: string[], _opts: object, cmd: Command) => { + const options = cmd.optsWithGlobals(); + await runEval(prompts, options); + }); +} + +interface BrandContext { + name?: string; + mentionBrand?: MentionBrand; + competitors: OnboardingCompetitor[]; +} + +async function runEval(positionalPrompts: string[], options: EvalOptions): Promise { + routeLibraryLogsToStderr(); + const format = parseFormat(options.format); + const runsPerTarget = parsePositiveInt(options.runs, "runs"); + const concurrency = parsePositiveInt(options.concurrency, "concurrency"); + + const loaded = await loadElmoEnv(options.dir); + const targets = resolveTargets(options.model); + + const { prompts, brand } = await gatherInputs(positionalPrompts, options); + if (prompts.length === 0) { + throw new Error( + "No prompts to evaluate. Pass them as arguments, with --prompt/--prompts-file, or via --brand-file.", + ); + } + + if (!brand.mentionBrand && (options.brandDomain || options.alias?.length)) { + log.warn("--brand-domain/--alias were ignored: no brand name was provided (pass --brand or --brand-file)."); + } + + const mentionBrand = brand.mentionBrand; + const mentionCompetitors: MentionCompetitor[] = toMentionCompetitors(brand.competitors); + const reportCompetitors: ReportCompetitor[] = toReportCompetitors(brand.competitors); + + log.step( + `Evaluating ${prompts.length} prompt(s) × ${targets.length} target(s) × ${runsPerTarget} run(s) = ${prompts.length * targets.length * runsPerTarget} calls…`, + ); + + // Flatten every (prompt, target, run) into one task list for the pool. + interface Task { + promptIndex: number; + prompt: string; + target: ResolvedTarget; + runIndex: number; + } + const tasks: Task[] = []; + for (let pi = 0; pi < prompts.length; pi++) { + for (const target of targets) { + for (let r = 1; r <= runsPerTarget; r++) { + tasks.push({ promptIndex: pi, prompt: prompts[pi], target, runIndex: r }); + } + } + } + + let completed = 0; + const taskResults = await mapPool(tasks, concurrency, async (task) => { + const run = await runOne(task.prompt, task.target, task.runIndex, mentionBrand, mentionCompetitors); + completed++; + process.stderr.write(`\r${" ".repeat(40)}\r`); + process.stderr.write(` ${completed}/${tasks.length} done`); + return { task, run }; + }); + process.stderr.write("\n"); + + // Re-assemble nested prompt → target → runs structure. + const promptResults: EvalPromptResult[] = prompts.map((prompt, index) => { + const targetResults: EvalTargetResult[] = targets.map((target) => { + const runs = taskResults + .filter((tr) => tr.task.promptIndex === index && tr.task.target.label === target.label) + .sort((a, b) => a.task.runIndex - b.task.runIndex) + .map((tr) => tr.run); + return { label: target.label, model: target.config.model, provider: target.config.provider, runs }; + }); + + // Per-prompt SoV across every run of every target. + const reportRuns: ReportPromptRun[] = targetResults.flatMap((t) => + t.runs + .filter((r) => !r.error && r.brandMentioned !== null) + .map((r) => ({ + promptId: String(index), + brandMentioned: Boolean(r.brandMentioned), + competitorsMentioned: r.competitorsMentioned, + })), + ); + const sov = reportRuns.length ? computePromptSoV(String(index), reportRuns, reportCompetitors).sov : null; + + return { index: index + 1, prompt, tags: [], targets: targetResults, sov }; + }); + + const report = buildReport( + promptResults, + targets.map((t) => t.label), + runsPerTarget, + brand, + reportCompetitors, + ); + + // ── stdout summary ────────────────────────────────────────────────────── + printSummary(report); + + // ── files ──────────────────────────────────────────────────────────────── + if (!options.stdout) { + await writeArtifacts(options.output, report, format, brand); + log.success( + `Wrote responses/, citations/mentions/share-of-voice/fan-out.${format}, summary.md and index.html to ${options.output}`, + ); + log.info(`Open the report: ${path.join(options.output, "index.html")}`); + } + + if (loaded.configDir) { + await trackCliEvent(loaded.configDir, "cli_lab_eval", { + prompt_count: prompts.length, + target_count: targets.length, + runs_per_target: runsPerTarget, + has_brand: Boolean(mentionBrand), + competitor_count: brand.competitors.length, + to_stdout: Boolean(options.stdout), + format, + }); + } +} + +async function runOne( + prompt: string, + target: ResolvedTarget, + runIndex: number, + mentionBrand: MentionBrand | undefined, + mentionCompetitors: MentionCompetitor[], +): Promise { + try { + const result = await target.provider.run(target.config.model, prompt, { + webSearch: target.config.webSearch, + version: target.config.version, + }); + const text = result.textContent ?? ""; + const mentions = mentionBrand ? analyzeMentions(text, mentionBrand, mentionCompetitors) : null; + const webQueries = dedupeFanout(result.webQueries ?? [], prompt); + return { + runIndex, + responseMarkdown: text, + brandMentioned: mentions ? mentions.brandMentioned : null, + competitorsMentioned: mentions ? mentions.competitorsMentioned : [], + citations: (result.citations ?? []).map((c) => ({ + url: c.url, + title: c.title, + domain: c.domain, + citationIndex: c.citationIndex, + })), + webQueries, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn(`run failed (${target.label}, run ${runIndex}): ${message}`); + return { + runIndex, + responseMarkdown: "", + brandMentioned: null, + competitorsMentioned: [], + citations: [], + webQueries: [], + error: message, + }; + } +} + +/** Drop the "unavailable" sentinel and queries that just echo the prompt verbatim. */ +function dedupeFanout(queries: string[], prompt: string): string[] { + const promptLower = prompt.trim().toLowerCase(); + const seen = new Set(); + const out: string[] = []; + for (const q of queries) { + const norm = q.trim().toLowerCase(); + if (!norm || norm === WEB_QUERIES_UNAVAILABLE || norm === promptLower) continue; + if (seen.has(norm)) continue; + seen.add(norm); + out.push(q.trim()); + } + return out; +} + +function buildReport( + prompts: EvalPromptResult[], + targetLabels: string[], + runsPerTarget: number, + brand: BrandContext, + reportCompetitors: ReportCompetitor[], +): EvalReport { + const allRuns: ReportPromptRun[] = prompts.flatMap((p) => + p.targets.flatMap((t) => + t.runs + .filter((r) => !r.error && r.brandMentioned !== null) + .map((r) => ({ + promptId: String(p.index), + brandMentioned: Boolean(r.brandMentioned), + competitorsMentioned: r.competitorsMentioned, + })), + ), + ); + const overallSov = brand.mentionBrand ? computeOverallSoV(allRuns, reportCompetitors) : null; + const competitorSov = brand.mentionBrand ? computeCompetitorSoVs(allRuns, reportCompetitors) : []; + + let responses = 0; + let citations = 0; + let fanoutQueries = 0; + for (const p of prompts) { + for (const t of p.targets) { + for (const r of t.runs) { + if (!r.error) responses++; + citations += r.citations.length; + fanoutQueries += r.webQueries.length; + } + } + } + + return { + brandName: brand.name, + generatedAt: new Date().toISOString(), + runsPerTarget, + targetLabels, + prompts, + overallSov, + competitorSov, + totals: { prompts: prompts.length, targets: targetLabels.length, responses, citations, fanoutQueries }, + }; +} + +// ── Output ──────────────────────────────────────────────────────────────────── + +function printSummary(report: EvalReport): void { + const rows: Row[] = []; + for (const p of report.prompts) { + for (const t of p.targets) { + const ok = t.runs.filter((r) => !r.error); + const brandHits = ok.filter((r) => r.brandMentioned === true).length; + const withBrand = ok.filter((r) => r.brandMentioned !== null).length; + rows.push({ + n: pad(p.index), + prompt: p.prompt, + target: t.label, + runs: ok.length, + brand_mention_rate: withBrand ? `${Math.round((brandHits / withBrand) * 100)}%` : "", + sov: p.sov === null ? "" : `${p.sov}%`, + citations: ok.reduce((s, r) => s + r.citations.length, 0), + fanout: ok.reduce((s, r) => s + r.webQueries.length, 0), + }); + } + } + printStdout(toCsv(rows, ["n", "prompt", "target", "runs", "brand_mention_rate", "sov", "citations", "fanout"])); + + // Fan-out as multiple lines of output, grouped by target. + const fanoutLines = formatFanoutLines(report); + if (fanoutLines) { + printStdout(`\n# query fan-out\n${fanoutLines}`); + } +} + +function formatFanoutLines(report: EvalReport): string { + const byTarget = new Map>(); + for (const p of report.prompts) { + for (const t of p.targets) { + const counts = byTarget.get(t.label) ?? new Map(); + for (const r of t.runs) { + for (const q of r.webQueries) counts.set(q, (counts.get(q) ?? 0) + 1); + } + if (counts.size) byTarget.set(t.label, counts); + } + } + const blocks: string[] = []; + for (const [label, counts] of byTarget) { + const lines = [...counts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([q, n]) => ` ${q}${n > 1 ? ` (×${n})` : ""}`); + blocks.push(`# ${label}\n${lines.join("\n")}`); + } + return blocks.join("\n"); +} + +async function writeArtifacts(dir: string, report: EvalReport, format: DataFormat, brand: BrandContext): Promise { + await ensureDir(dir); + + // Clear our `responses/` tree first so a re-run (fewer runs/targets, or an + // edited prompt that changes the slug) can't leave orphaned markdown that + // looks like part of this run. Only our own subdir is removed; the rest of + // the output dir (which defaults to the cwd) is left untouched. + const responsesDir = path.join(dir, "responses"); + await fs.rm(responsesDir, { recursive: true, force: true }); + + const citationRows: Row[] = []; + const mentionRows: Row[] = []; + const fanoutRows: Row[] = []; + const fullRuns: FullPromptRun[] = []; + + for (const p of report.prompts) { + const promptDir = path.join(responsesDir, `${pad(p.index)}-${slugify(p.prompt)}`); + for (const t of p.targets) { + for (const r of t.runs) { + const base = `${t.model}__${t.provider}__run-${r.runIndex}`; + const header = [ + `# ${p.prompt}`, + "", + `- target: \`${t.label}\``, + `- run: ${r.runIndex}`, + r.brandMentioned === null ? "" : `- brand mentioned: ${r.brandMentioned ? "yes" : "no"}`, + r.competitorsMentioned.length ? `- competitors mentioned: ${r.competitorsMentioned.join(", ")}` : "", + "", + "---", + "", + ] + .filter((l) => l !== "") + .join("\n"); + const body = r.error ? `> Run failed: ${r.error}` : r.responseMarkdown || "_(empty response)_"; + await writeText(promptDir, `${base}.md`, `${header}\n${body}\n`); + + for (const c of r.citations) { + citationRows.push({ + prompt_n: p.index, + prompt: p.prompt, + model: t.model, + provider: t.provider, + run: r.runIndex, + citation_index: c.citationIndex, + url: c.url, + domain: c.domain, + title: c.title ?? "", + }); + } + mentionRows.push({ + prompt_n: p.index, + prompt: p.prompt, + model: t.model, + provider: t.provider, + run: r.runIndex, + brand_mentioned: r.brandMentioned === null ? "" : r.brandMentioned, + competitors_mentioned: r.competitorsMentioned, + error: r.error ?? "", + }); + for (const q of r.webQueries) { + fanoutRows.push({ prompt_n: p.index, prompt: p.prompt, model: t.model, provider: t.provider, query: q }); + } + if (!r.error && r.brandMentioned !== null) { + fullRuns.push({ + promptId: String(p.index), + promptValue: p.prompt, + brandMentioned: r.brandMentioned, + competitorsMentioned: r.competitorsMentioned, + webQueries: r.webQueries, + textContent: r.responseMarkdown, + model: t.model, + }); + } + } + } + } + + await writeStructured( + dir, + "citations", + citationRows, + ["prompt_n", "prompt", "model", "provider", "run", "citation_index", "url", "domain", "title"], + format, + ); + await writeStructured( + dir, + "mentions", + mentionRows, + ["prompt_n", "prompt", "model", "provider", "run", "brand_mentioned", "competitors_mentioned", "error"], + format, + ); + await writeStructured(dir, "fan-out", fanoutRows, ["prompt_n", "prompt", "model", "provider", "query"], format); + + const sovRows: Row[] = []; + if (report.overallSov !== null) + sovRows.push({ scope: "brand", name: report.brandName ?? "brand", sov: report.overallSov, mentions: "" }); + for (const c of report.competitorSov) + sovRows.push({ scope: "competitor", name: c.name, sov: c.sov, mentions: c.mentionCount }); + for (const p of report.prompts) + sovRows.push({ scope: "prompt", name: `${pad(p.index)} ${p.prompt}`, sov: p.sov ?? "", mentions: "" }); + await writeStructured(dir, "share-of-voice", sovRows, ["scope", "name", "sov", "mentions"], format); + + const contentGaps = brand.mentionBrand ? findContentGaps(fullRuns, 10) : []; + await writeJson(dir, "run.json", { + generatedAt: report.generatedAt, + brandName: report.brandName, + targets: report.targetLabels, + runsPerTarget: report.runsPerTarget, + totals: report.totals, + overallSov: report.overallSov, + competitorSov: report.competitorSov, + contentGaps, + }); + + await writeText(dir, "summary.md", buildSummaryMarkdown(report, contentGaps)); + await writeText(dir, "index.html", buildEvalReportHtml(report)); +} + +function buildSummaryMarkdown( + report: EvalReport, + contentGaps: { promptValue: string; competitorsMentioned: string[] }[], +): string { + const lines: string[] = []; + lines.push(`# Elmo eval${report.brandName ? ` — ${report.brandName}` : ""}`, ""); + lines.push(`Generated ${report.generatedAt}`, ""); + lines.push( + `- Prompts: ${report.totals.prompts}`, + `- Targets: ${report.targetLabels.join(", ")}`, + `- Runs per target: ${report.runsPerTarget}`, + `- Responses: ${report.totals.responses} · Citations: ${report.totals.citations} · Fan-out queries: ${report.totals.fanoutQueries}`, + "", + ); + if (report.overallSov !== null) { + lines.push(`## Share of voice`, "", `- **${report.brandName ?? "Brand"}: ${report.overallSov}%**`); + for (const c of report.competitorSov) lines.push(`- ${c.name}: ${c.sov}% (${c.mentionCount})`); + lines.push(""); + } + if (contentGaps.length) { + lines.push(`## Content gaps (competitors cited, brand absent)`, ""); + for (const g of contentGaps) lines.push(`- ${g.promptValue} — ${g.competitorsMentioned.join(", ")}`); + lines.push(""); + } + lines.push(`Open \`index.html\` to browse every response.`, ""); + return lines.join("\n"); +} + +// ── Input gathering ──────────────────────────────────────────────────────────── + +async function gatherInputs( + positional: string[], + options: EvalOptions, +): Promise<{ prompts: string[]; brand: BrandContext }> { + const explicit: string[] = [...positional]; + if (options.prompt) explicit.push(...options.prompt); + if (options.promptsFile) explicit.push(...(await readPromptsFile(options.promptsFile))); + // Treat a lone "-" positional as a stdin request. + const wantsStdin = explicit.includes("-"); + const cleaned = explicit + .filter((p) => p !== "-") + .map((p) => p.trim()) + .filter(Boolean); + if (wantsStdin || (cleaned.length === 0 && !options.brandFile && !process.stdin.isTTY)) { + cleaned.push(...(await readStdinLines())); + } + + let pack: BrandPack | undefined; + if (options.brandFile) pack = await readBrandPack(options.brandFile); + + const prompts = dedupePrompts(cleaned.length ? cleaned : (pack?.prompts ?? []).map((p) => p.prompt)); + + const brand = buildBrandContext(pack, options); + return { prompts, brand }; +} + +function buildBrandContext(pack: BrandPack | undefined, options: EvalOptions): BrandContext { + const flagCompetitors = (options.competitor ?? []).map(parseCompetitor); + if (pack) { + const competitors = [...pack.competitors, ...flagCompetitors]; + return { + name: options.brand ?? pack.brandName, + mentionBrand: toMentionBrand({ + brandName: options.brand ?? pack.brandName, + website: options.brandDomain ?? pack.website, + aliases: options.alias ?? pack.aliases, + additionalDomains: pack.additionalDomains, + }), + competitors, + }; + } + if (options.brand || flagCompetitors.length) { + return { + name: options.brand, + mentionBrand: options.brand + ? { name: options.brand, website: options.brandDomain, aliases: options.alias } + : undefined, + competitors: flagCompetitors, + }; + } + return { competitors: [] }; +} + +function parseCompetitor(spec: string): OnboardingCompetitor { + const idx = spec.indexOf(":"); + const name = (idx === -1 ? spec : spec.slice(0, idx)).trim(); + const domain = idx === -1 ? "" : spec.slice(idx + 1).trim(); + return { name, domains: domain ? [domain] : [], aliases: [] }; +} + +function dedupePrompts(prompts: string[]): string[] { + const seen = new Set(); + const out: string[] = []; + for (const p of prompts) { + const key = p.trim().toLowerCase(); + if (!key || seen.has(key)) continue; + seen.add(key); + out.push(p.trim()); + } + return out; +} + +async function readPromptsFile(file: string): Promise { + if (file === "-") return readStdinLines(); + const contents = await fs.readFile(path.resolve(process.cwd(), file), "utf8"); + return parsePromptLines(contents); +} + +async function readStdinLines(): Promise { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) chunks.push(chunk as Buffer); + return parsePromptLines(Buffer.concat(chunks).toString("utf8")); +} + +function parsePromptLines(contents: string): string[] { + return contents + .split("\n") + .map((l) => l.trim()) + .filter((l) => l && !l.startsWith("#")); +} + +function parsePositiveInt(value: string, name: string): number { + const n = Number(value); + if (!Number.isInteger(n) || n < 1) { + throw new Error(`--${name} must be a positive integer (got "${value}")`); + } + return n; +} diff --git a/apps/cli/src/commands/plan.ts b/apps/cli/src/commands/plan.ts new file mode 100644 index 00000000..d6193d3e --- /dev/null +++ b/apps/cli/src/commands/plan.ts @@ -0,0 +1,271 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { runStructuredResearchPrompt } from "@workspace/lib/onboarding"; +import { getWebsiteExcerpt } from "@workspace/lib/website-excerpt"; +import type { Command } from "commander"; +import { z } from "zod"; +import { type BrandPack, type PlanSuggestion, readBrandPack } from "../core/brand-pack.js"; +import { loadElmoEnv } from "../core/env.js"; +import { parseFormat, printStdout, type Row, writeJson, writeStructured, writeText } from "../core/output.js"; +import { applyResearchTarget } from "../core/targets.js"; +import { log, routeLibraryLogsToStderr } from "../core/ui.js"; +import { trackCliEvent } from "../telemetry.js"; + +interface PlanOptions { + model?: string; + website?: string; + brandFile?: string; + evalDir?: string; + maxBytes: string; + output: string; + format: string; + stdout?: boolean; + dir?: string; +} + +// Text-ish files we'll read when a directory is passed. +const TEXT_EXTENSIONS = new Set([ + ".md", + ".mdx", + ".markdown", + ".txt", + ".html", + ".htm", + ".json", + ".csv", + ".rst", + ".text", +]); +const DEFAULT_MAX_BYTES = 200_000; + +const planSchema = z.object({ + suggestions: z + .array( + z.object({ + title: z.string().describe("Short, action-oriented title for the recommendation."), + category: z + .string() + .describe("A short category, e.g. content, structure, schema, authority, technical, comparison."), + priority: z.enum(["high", "medium", "low"]).describe("Impact/effort priority."), + recommendation: z.string().describe("Concrete, specific guidance — what to do and why it helps AI visibility."), + evidence: z.string().optional().describe("What in the provided context motivates this (quote or reference)."), + }), + ) + .describe("Prioritized AEO/answer-engine-optimization recommendations grounded in the provided context."), + competitors: z + .array( + z.object({ + name: z.string(), + domains: z.array(z.string()).describe("Hostnames only, no protocol/www."), + aliases: z.array(z.string()), + }), + ) + .describe("Direct competitors implied by the context (for tracking). Empty if uncertain."), +}); + +type PlanResult = z.infer; + +export function registerPlan(lab: Command): void { + lab + .command("plan") + .description("generate AEO recommendations (+ competitors) from your content, optionally grounded in an eval") + .argument("[paths...]", "files or directories to use as context") + .option("-m, --model ", "research provider, model:provider[:version] (direct API only)") + .option("--website ", "also pull a short excerpt from this site for context") + .option("--brand-file ", "brand pack JSON for brand/competitor context (augmented on output)") + .option("--eval-dir ", "an `elmo lab eval` output directory to ground recommendations in") + .option("--max-bytes ", "cap on context bytes read from files", String(DEFAULT_MAX_BYTES)) + .option("-o, --output ", "directory to write artifacts to", ".") + .option("--format ", "structured output format", "csv") + .option("--stdout", "print to stdout only; do not write files") + .action(async (paths: string[], _opts: object, cmd: Command) => { + const options = cmd.optsWithGlobals(); + await runPlan(paths, options); + }); +} + +async function runPlan(paths: string[], options: PlanOptions): Promise { + routeLibraryLogsToStderr(); + const format = parseFormat(options.format); + const maxBytes = Number(options.maxBytes) || DEFAULT_MAX_BYTES; + + const loaded = await loadElmoEnv(options.dir); + applyResearchTarget(options.model); + + const pack = options.brandFile ? await readBrandPack(options.brandFile) : undefined; + + const context = await assembleContext(paths, options, maxBytes); + if (!context.trim()) { + throw new Error("No context to plan from. Pass files/directories, --website, --brand-file, or --eval-dir."); + } + + log.step("Generating AEO recommendations…"); + const prompt = buildPlanPrompt(context, pack); + const result = await runStructuredResearchPrompt(prompt, planSchema); + + const markdown = renderPlanMarkdown(result, pack); + printStdout(markdown); + + if (!options.stdout) { + const dir = options.output; + await writeText(dir, "plan.md", markdown); + const suggestionRows: Row[] = result.suggestions.map((s, i) => ({ + n: i + 1, + priority: s.priority, + category: s.category, + title: s.title, + recommendation: s.recommendation, + evidence: s.evidence ?? "", + })); + await writeStructured( + dir, + "suggestions", + suggestionRows, + ["n", "priority", "category", "title", "recommendation", "evidence"], + format, + ); + const competitorRows: Row[] = result.competitors.map((c) => ({ + name: c.name, + domains: c.domains, + aliases: c.aliases, + })); + await writeStructured(dir, "competitors", competitorRows, ["name", "domains", "aliases"], format); + await writeJson(dir, "brand.json", augmentBrandPack(pack, result, options)); + log.success(`Wrote plan.md, suggestions.${format}, competitors.${format}, brand.json to ${dir}.`); + } + + if (loaded.configDir) { + await trackCliEvent(loaded.configDir, "cli_lab_plan", { + suggestion_count: result.suggestions.length, + competitor_count: result.competitors.length, + has_model: Boolean(options.model), + has_eval: Boolean(options.evalDir), + to_stdout: Boolean(options.stdout), + format, + }); + } +} + +async function assembleContext(paths: string[], options: PlanOptions, maxBytes: number): Promise { + const parts: string[] = []; + let budget = maxBytes; + + for (const p of paths) { + const resolved = path.resolve(process.cwd(), p); + const stat = await fs.stat(resolved).catch(() => null); + if (!stat) { + log.warn(`Skipping ${p}: not found.`); + continue; + } + const files = stat.isDirectory() ? await walkTextFiles(resolved) : [resolved]; + for (const file of files) { + if (budget <= 0) break; + const content = await fs.readFile(file, "utf8").catch(() => ""); + if (!content.trim()) continue; + const slice = content.slice(0, budget); + budget -= slice.length; + parts.push(`<<< FILE: ${path.relative(process.cwd(), file)} >>>\n${slice}`); + } + } + + if (options.website) { + const excerpt = await getWebsiteExcerpt(options.website).catch(() => ""); + if (excerpt.trim()) parts.push(`<<< WEBSITE: ${options.website} >>>\n${excerpt}`); + } + + if (options.evalDir) { + const evalContext = await readEvalContext(options.evalDir); + if (evalContext) parts.push(`<<< PRIOR ELMO EVAL FINDINGS >>>\n${evalContext}`); + } + + return parts.join("\n\n"); +} + +async function walkTextFiles(dir: string, depth = 0): Promise { + if (depth > 6) return []; + const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []); + const out: string[] = []; + for (const entry of entries) { + if (entry.name.startsWith(".") || entry.name === "node_modules") continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + out.push(...(await walkTextFiles(full, depth + 1))); + } else if (TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) { + out.push(full); + } + } + return out.sort(); +} + +/** Pull the human-readable rollup from a prior eval to ground the plan. */ +async function readEvalContext(evalDir: string): Promise { + const dir = path.resolve(process.cwd(), evalDir); + const summary = await fs.readFile(path.join(dir, "summary.md"), "utf8").catch(() => ""); + if (summary.trim()) return summary; + const runJson = await fs.readFile(path.join(dir, "run.json"), "utf8").catch(() => ""); + return runJson.trim(); +} + +function buildPlanPrompt(context: string, pack: BrandPack | undefined): string { + const brandLine = pack?.brandName + ? `The brand is ${pack.brandName}${pack.website ? ` (${pack.website})` : ""}.` + : "Infer the brand from the context."; + return `You are an expert in Answer Engine Optimization (AEO) / Generative Engine Optimization — getting a brand cited and recommended by AI assistants like ChatGPT, Google AI Mode, Perplexity, and Claude. + +${brandLine} + +Using ONLY the context below (plus web search to verify facts), produce a prioritized set of concrete AEO recommendations to improve how often and how favorably AI answer engines surface this brand. Favor specific, implementable actions over generic advice. Ground each recommendation in the provided context, and prefer high-leverage moves: clear comparison/answer content, structured data, authoritative citations and mentions, fixing gaps where competitors are cited but the brand is not, and topical coverage of the unbranded queries buyers ask. + +Also list the direct competitors implied by the context so they can be tracked. + +CONTEXT: +${context}`; +} + +function renderPlanMarkdown(result: PlanResult, pack: BrandPack | undefined): string { + const lines: string[] = []; + lines.push(`# AEO plan${pack?.brandName ? ` — ${pack.brandName}` : ""}`, ""); + const order = { high: 0, medium: 1, low: 2 } as Record; + const sorted = [...result.suggestions].sort((a, b) => (order[a.priority] ?? 3) - (order[b.priority] ?? 3)); + for (const s of sorted) { + lines.push(`## [${s.priority.toUpperCase()}] ${s.title}`); + lines.push(`*${s.category}*`, ""); + lines.push(s.recommendation); + if (s.evidence) lines.push("", `> ${s.evidence}`); + lines.push(""); + } + if (result.competitors.length) { + lines.push(`## Competitors to track`, ""); + for (const c of result.competitors) { + lines.push(`- **${c.name}**${c.domains.length ? ` — ${c.domains.join(", ")}` : ""}`); + } + lines.push(""); + } + return lines.join("\n"); +} + +function augmentBrandPack(pack: BrandPack | undefined, result: PlanResult, options: PlanOptions): BrandPack { + const suggestions: PlanSuggestion[] = result.suggestions.map((s) => ({ + title: s.title, + category: s.category, + priority: s.priority, + recommendation: s.recommendation, + evidence: s.evidence, + })); + const base: BrandPack = pack ?? { + brandName: "", + website: options.website ?? "", + aliases: [], + additionalDomains: [], + competitors: [], + prompts: [], + }; + // Merge competitors by name, preferring existing entries. + const byName = new Map(base.competitors.map((c) => [c.name.toLowerCase(), c])); + for (const c of result.competitors) { + if (!byName.has(c.name.toLowerCase())) { + byName.set(c.name.toLowerCase(), { name: c.name, domains: c.domains, aliases: c.aliases }); + } + } + return { ...base, competitors: [...byName.values()], suggestions }; +} diff --git a/apps/cli/src/core/brand-pack.test.ts b/apps/cli/src/core/brand-pack.test.ts new file mode 100644 index 00000000..28f5ac13 --- /dev/null +++ b/apps/cli/src/core/brand-pack.test.ts @@ -0,0 +1,66 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { OnboardingSuggestion } from "@workspace/lib/onboarding"; +import { describe, expect, it } from "vitest"; +import { readBrandPack, suggestionToBrandPack, toMentionBrand, toReportCompetitors } from "./brand-pack"; + +const suggestion: OnboardingSuggestion = { + brandName: "Nike", + website: "nike.com", + additionalDomains: ["nike.co.uk"], + aliases: ["Nike Inc"], + competitors: [{ name: "Adidas", domains: ["adidas.com", "adidas.de"], aliases: [] }], + suggestedPrompts: [ + { prompt: "best running shoes", tags: ["footwear"] }, + { prompt: "nike alternative", tags: ["brand"] }, + ], +}; + +describe("suggestionToBrandPack", () => { + it("maps suggestedPrompts to prompts", () => { + const pack = suggestionToBrandPack(suggestion); + expect(pack.prompts).toHaveLength(2); + expect(pack.prompts[0].prompt).toBe("best running shoes"); + expect(pack.competitors[0].name).toBe("Adidas"); + }); +}); + +describe("toReportCompetitors", () => { + it("uses the first domain", () => { + expect(toReportCompetitors(suggestion.competitors)).toEqual([{ name: "Adidas", domain: "adidas.com" }]); + }); +}); + +describe("toMentionBrand", () => { + it("maps brand pack fields to the mention shape", () => { + const pack = suggestionToBrandPack(suggestion); + expect(toMentionBrand(pack)).toEqual({ + name: "Nike", + website: "nike.com", + aliases: ["Nike Inc"], + additionalDomains: ["nike.co.uk"], + }); + }); +}); + +describe("readBrandPack", () => { + it("round-trips a written pack and tolerates suggestedPrompts", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "elmo-pack-")); + const file = path.join(dir, "brand.json"); + await fs.writeFile(file, JSON.stringify(suggestion), "utf8"); + const pack = await readBrandPack(file); + expect(pack.brandName).toBe("Nike"); + // `suggestedPrompts` (onboarding shape) is read as `prompts`. + expect(pack.prompts.map((p) => p.prompt)).toEqual(["best running shoes", "nike alternative"]); + await fs.rm(dir, { recursive: true, force: true }); + }); + + it("throws when neither brandName nor website is present", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "elmo-pack-")); + const file = path.join(dir, "bad.json"); + await fs.writeFile(file, JSON.stringify({ prompts: [] }), "utf8"); + await expect(readBrandPack(file)).rejects.toThrow(/missing both brandName and website/); + await fs.rm(dir, { recursive: true, force: true }); + }); +}); diff --git a/apps/cli/src/core/brand-pack.ts b/apps/cli/src/core/brand-pack.ts new file mode 100644 index 00000000..a6ed8c73 --- /dev/null +++ b/apps/cli/src/core/brand-pack.ts @@ -0,0 +1,87 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { MentionBrand, MentionCompetitor } from "@workspace/lib/mentions"; +import type { OnboardingCompetitor, OnboardingPrompt, OnboardingSuggestion } from "@workspace/lib/onboarding"; +import type { ReportCompetitor } from "@workspace/lib/report-metrics"; + +export interface PlanSuggestion { + title: string; + category: string; + priority: string; + recommendation: string; + evidence?: string; +} + +/** + * The pipe artifact shared between `lab` commands. `brainstorm` and `plan` + * write it (`brand.json`); `eval` reads it for the brand + competitor context + * it needs to compute mentions and share-of-voice. Shaped like the onboarding + * `OnboardingSuggestion` but with `prompts` (not `suggestedPrompts`) and an + * optional `suggestions` list contributed by `plan`. + */ +export interface BrandPack { + brandName: string; + website: string; + aliases: string[]; + additionalDomains: string[]; + competitors: OnboardingCompetitor[]; + prompts: OnboardingPrompt[]; + suggestions?: PlanSuggestion[]; +} + +export function suggestionToBrandPack(s: OnboardingSuggestion): BrandPack { + return { + brandName: s.brandName, + website: s.website, + aliases: s.aliases ?? [], + additionalDomains: s.additionalDomains ?? [], + competitors: s.competitors ?? [], + prompts: s.suggestedPrompts ?? [], + }; +} + +/** Read a brand pack, tolerating either `prompts` or `suggestedPrompts` keys. */ +export async function readBrandPack(filePath: string): Promise { + const resolved = path.resolve(process.cwd(), filePath); + let parsed: unknown; + try { + parsed = JSON.parse(await fs.readFile(resolved, "utf8")); + } catch (err) { + throw new Error(`Could not read brand pack at ${resolved}: ${err instanceof Error ? err.message : String(err)}`); + } + const obj = (parsed ?? {}) as Record; + const prompts = (obj.prompts ?? obj.suggestedPrompts ?? []) as OnboardingPrompt[]; + if (!obj.brandName && !obj.website) { + throw new Error(`Brand pack at ${resolved} is missing both brandName and website.`); + } + return { + brandName: String(obj.brandName ?? ""), + website: String(obj.website ?? ""), + aliases: (obj.aliases as string[]) ?? [], + additionalDomains: (obj.additionalDomains as string[]) ?? [], + competitors: (obj.competitors as OnboardingCompetitor[]) ?? [], + prompts, + suggestions: obj.suggestions as PlanSuggestion[] | undefined, + }; +} + +/** Adapt a brand pack to the shared mention-detection brand shape. */ +export function toMentionBrand( + pack: Pick, +): MentionBrand { + return { + name: pack.brandName, + website: pack.website || undefined, + aliases: pack.aliases, + additionalDomains: pack.additionalDomains, + }; +} + +export function toMentionCompetitors(competitors: OnboardingCompetitor[]): MentionCompetitor[] { + return competitors.map((c) => ({ name: c.name, domains: c.domains, aliases: c.aliases })); +} + +/** Adapt to the report-metrics competitor shape (single primary domain). */ +export function toReportCompetitors(competitors: OnboardingCompetitor[]): ReportCompetitor[] { + return competitors.map((c) => ({ name: c.name, domain: c.domains[0] ?? "" })); +} diff --git a/apps/cli/src/core/env.ts b/apps/cli/src/core/env.ts new file mode 100644 index 00000000..4dd037cc --- /dev/null +++ b/apps/cli/src/core/env.ts @@ -0,0 +1,66 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { parse as parseDotenv } from "dotenv"; +import { log } from "./ui.js"; + +/** Default config directory written by `elmo init`. */ +const CONFIG_HOME = path.join(os.homedir(), ".elmo"); + +async function fileExists(target: string): Promise { + try { + await fs.access(target); + return true; + } catch { + return false; + } +} + +export interface LoadedEnv { + /** + * Directory the `.env` was loaded from (used for telemetry). Undefined when + * no Elmo `.env` was found and we're running purely off the ambient + * environment. + */ + configDir?: string; + /** Whether an Elmo-managed `.env` file was found and loaded. */ + loaded: boolean; +} + +/** + * Load the provider keys + SCRAPE_TARGETS that `elmo init` wrote, so the `lab` + * commands can reach the same providers the deployment uses. Values are merged + * into `process.env` **without** clobbering anything already exported in the + * shell (ambient env wins; the file fills the gaps), because provider + * implementations read their keys lazily from `process.env`. + * + * Resolution: `--dir` if given, else `~/.elmo`. With an explicit `--dir` that + * has no `.env` we throw; with the default we silently fall back to the ambient + * environment so users who export keys themselves can still run the commands. + * + * This never reads or requires `DATABASE_URL` — the lab commands never touch a + * database. + */ +export async function loadElmoEnv(explicitDir?: string): Promise { + const dir = explicitDir ? path.resolve(process.cwd(), explicitDir) : CONFIG_HOME; + const envPath = path.join(dir, ".env"); + + if (!(await fileExists(envPath))) { + if (explicitDir) { + throw new Error( + `No .env found at ${envPath}. Run \`elmo init --dir ${explicitDir}\` first, or export the provider keys yourself.`, + ); + } + log.warn(`No Elmo config found at ${envPath}; using the current environment for provider keys.`); + return { loaded: false }; + } + + const contents = await fs.readFile(envPath, "utf8"); + const values = parseDotenv(contents); + for (const [key, value] of Object.entries(values)) { + if (process.env[key] === undefined) { + process.env[key] = value; + } + } + return { configDir: dir, loaded: true }; +} diff --git a/apps/cli/src/core/output.test.ts b/apps/cli/src/core/output.test.ts new file mode 100644 index 00000000..30b447cb --- /dev/null +++ b/apps/cli/src/core/output.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { pad, parseFormat, serialize, slugify, toCsv, toJsonl } from "./output"; + +describe("toCsv", () => { + it("renders a header and rows", () => { + const csv = toCsv([{ a: 1, b: "x" }], ["a", "b"]); + expect(csv).toBe("a,b\n1,x"); + }); + + it("escapes commas, quotes, and newlines", () => { + const csv = toCsv([{ v: 'a,"b"\nc' }], ["v"]); + expect(csv).toBe('v\n"a,""b""\nc"'); + }); + + it("joins arrays with '; '", () => { + const csv = toCsv([{ tags: ["a", "b"] }], ["tags"]); + expect(csv).toBe("tags\na; b"); + }); + + it("renders missing columns as empty", () => { + expect(toCsv([{ a: 1 }], ["a", "b"])).toBe("a,b\n1,"); + }); + + it("neutralizes formula injection in text cells but not numbers", () => { + // prefixed with ' (no other special chars → not quoted) + expect(toCsv([{ q: "=danger" }], ["q"])).toBe("q\n'=danger"); + expect(toCsv([{ q: "+1-800-CALL" }], ["q"])).toBe("q\n'+1-800-CALL"); + // prefixed AND quoted when it also contains a comma + expect(toCsv([{ q: "=A1,B2" }], ["q"])).toBe(`q\n"'=A1,B2"`); + // negative numbers are emitted verbatim + expect(toCsv([{ n: -5 }], ["n"])).toBe("n\n-5"); + }); +}); + +describe("toJsonl", () => { + it("writes one JSON object per line", () => { + expect(toJsonl([{ a: 1 }, { b: 2 }])).toBe('{"a":1}\n{"b":2}'); + }); +}); + +describe("serialize", () => { + it("dispatches on format", () => { + const rows = [{ a: 1 }]; + expect(serialize(rows, ["a"], "csv")).toBe("a\n1"); + expect(serialize(rows, ["a"], "jsonl")).toBe('{"a":1}'); + }); +}); + +describe("parseFormat", () => { + it("defaults to csv and accepts jsonl", () => { + expect(parseFormat(undefined)).toBe("csv"); + expect(parseFormat("jsonl")).toBe("jsonl"); + expect(parseFormat("CSV")).toBe("csv"); + }); + it("rejects unknown formats", () => { + expect(() => parseFormat("yaml")).toThrow(/Unknown --format/); + }); +}); + +describe("slugify", () => { + it("produces filesystem-safe slugs", () => { + expect(slugify("Best Running Shoes!")).toBe("best-running-shoes"); + expect(slugify(" ")).toBe("prompt"); + }); + it("caps length without a trailing dash", () => { + expect(slugify("a".repeat(80), 10)).toBe("aaaaaaaaaa"); + }); +}); + +describe("pad", () => { + it("zero-pads to width 3", () => { + expect(pad(1)).toBe("001"); + expect(pad(42)).toBe("042"); + }); +}); diff --git a/apps/cli/src/core/output.ts b/apps/cli/src/core/output.ts new file mode 100644 index 00000000..29b50876 --- /dev/null +++ b/apps/cli/src/core/output.ts @@ -0,0 +1,120 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +export type DataFormat = "csv" | "jsonl"; + +export function parseFormat(value: string | undefined): DataFormat { + const v = (value ?? "csv").toLowerCase(); + if (v !== "csv" && v !== "jsonl") { + throw new Error(`Unknown --format "${value}". Use "csv" or "jsonl".`); + } + return v; +} + +export async function ensureDir(dir: string): Promise { + await fs.mkdir(dir, { recursive: true }); +} + +/** filesystem-safe, readable slug for prompt directories (e.g. 001-best-shoes). */ +export function slugify(value: string, maxLen = 48): string { + const slug = value + .toLowerCase() + .trim() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, maxLen) + .replace(/-+$/g, ""); + return slug || "prompt"; +} + +/** Zero-pad an index for stable directory ordering (1 -> "001"). */ +export function pad(n: number, width = 3): string { + return String(n).padStart(width, "0"); +} + +// ── CSV ────────────────────────────────────────────────────────────────────── + +function csvCell(value: unknown): string { + let str: string; + // Only text-origin cells can carry formula-injection payloads; numbers and + // booleans are emitted verbatim so legitimate values (e.g. negatives) aren't + // mangled. + let isText = true; + if (value === null || value === undefined) { + str = ""; + } else if (typeof value === "number" || typeof value === "boolean") { + str = String(value); + isText = false; + } else if (Array.isArray(value)) { + str = value.join("; "); + } else if (typeof value === "object") { + str = JSON.stringify(value); + } else { + str = String(value); + } + // Neutralize spreadsheet formula injection: prompts, citation titles, and + // fan-out queries are LLM/web-derived and may start with =, +, -, @, etc. + if (isText && /^[=+\-@\t\r]/.test(str)) { + str = `'${str}`; + } + if (/[",\n\r]/.test(str)) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; +} + +export type Row = Record; + +export function toCsv(rows: Row[], columns: string[]): string { + const header = columns.map(csvCell).join(","); + const body = rows.map((row) => columns.map((col) => csvCell(row[col])).join(",")); + return [header, ...body].join("\n"); +} + +export function toJsonl(rows: Row[]): string { + return rows.map((row) => JSON.stringify(row)).join("\n"); +} + +/** Serialize structured rows in the requested format (string only — no I/O). */ +export function serialize(rows: Row[], columns: string[], format: DataFormat): string { + return format === "csv" ? toCsv(rows, columns) : toJsonl(rows); +} + +/** + * Write structured rows to `/.` in the chosen format and + * return the path written. + */ +export async function writeStructured( + dir: string, + baseName: string, + rows: Row[], + columns: string[], + format: DataFormat, +): Promise { + await ensureDir(dir); + const ext = format === "csv" ? "csv" : "jsonl"; + const filePath = path.join(dir, `${baseName}.${ext}`); + await fs.writeFile(filePath, `${serialize(rows, columns, format)}\n`, "utf8"); + return filePath; +} + +export async function writeText(dir: string, name: string, content: string): Promise { + await ensureDir(dir); + const filePath = path.join(dir, name); + await fs.writeFile(filePath, content, "utf8"); + return filePath; +} + +export async function writeJson(dir: string, name: string, value: unknown): Promise { + return writeText(dir, name, `${JSON.stringify(value, null, 2)}\n`); +} + +// ── stdout printers ────────────────────────────────────────────────────────── + +export function printStdout(content: string): void { + process.stdout.write(content.endsWith("\n") ? content : `${content}\n`); +} + +export function printCsv(rows: Row[], columns: string[]): void { + printStdout(toCsv(rows, columns)); +} diff --git a/apps/cli/src/core/pool.ts b/apps/cli/src/core/pool.ts new file mode 100644 index 00000000..8a334725 --- /dev/null +++ b/apps/cli/src/core/pool.ts @@ -0,0 +1,25 @@ +/** + * Run `fn` over `items` with at most `concurrency` in flight at once. Results + * are returned in input order. Rejections propagate (callers that want + * per-item error capture should catch inside `fn`). + */ +export async function mapPool( + items: T[], + concurrency: number, + fn: (item: T, index: number) => Promise, +): Promise { + const results = new Array(items.length); + let next = 0; + const limit = Math.max(1, Math.min(concurrency, items.length || 1)); + + async function worker(): Promise { + while (true) { + const index = next++; + if (index >= items.length) return; + results[index] = await fn(items[index], index); + } + } + + await Promise.all(Array.from({ length: limit }, () => worker())); + return results; +} diff --git a/apps/cli/src/core/report-html.test.ts b/apps/cli/src/core/report-html.test.ts new file mode 100644 index 00000000..4a6e6280 --- /dev/null +++ b/apps/cli/src/core/report-html.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { buildEvalReportHtml, type EvalReport } from "./report-html"; + +function makeReport(overrides: Partial = {}): EvalReport { + return { + brandName: "Nike", + generatedAt: "2026-06-16T00:00:00.000Z", + runsPerTarget: 1, + targetLabels: ["chatgpt:brightdata:online"], + overallSov: 60, + competitorSov: [{ name: "Adidas", sov: 40, mentionCount: 2 }], + totals: { prompts: 1, targets: 1, responses: 1, citations: 1, fanoutQueries: 1 }, + prompts: [ + { + index: 1, + prompt: "best running shoes", + tags: ["footwear"], + sov: 60, + targets: [ + { + label: "chatgpt:brightdata:online", + model: "chatgpt", + provider: "brightdata", + runs: [ + { + runIndex: 1, + responseMarkdown: "**Nike** is great ", + brandMentioned: true, + competitorsMentioned: ["Adidas"], + citations: [{ url: "https://nike.com", title: "Nike", domain: "nike.com", citationIndex: 0 }], + webQueries: ["best running shoes 2026"], + }, + ], + }, + ], + }, + ], + ...overrides, + }; +} + +describe("buildEvalReportHtml", () => { + it("produces a self-contained document with no external resources", () => { + const html = buildEvalReportHtml(makeReport()); + expect(html.startsWith("")).toBe(true); + expect(html).toContain(" + + +
+
+

Elmo eval${report.brandName ? ` — ${escapeHtml(report.brandName)}` : ""}

+
${escapeHtml(report.generatedAt)} · ${report.runsPerTarget} run(s) per target · ${escapeHtml(report.targetLabels.join(", "))}
+
${stats}
+
${sovBars}
+
+ + ${report.prompts.map(renderPrompt).join("\n")} +
+ + + +`; +} diff --git a/apps/cli/src/core/targets.test.ts b/apps/cli/src/core/targets.test.ts new file mode 100644 index 00000000..dd26c0a4 --- /dev/null +++ b/apps/cli/src/core/targets.test.ts @@ -0,0 +1,55 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { applyResearchTarget, resolveTargets } from "./targets"; + +const ENV_KEYS = ["SCRAPE_TARGETS", "BRIGHTDATA_API_TOKEN", "ONBOARDING_LLM_TARGET"]; +let saved: Record; + +beforeEach(() => { + saved = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + for (const k of ENV_KEYS) delete process.env[k]; +}); +afterEach(() => { + for (const k of ENV_KEYS) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } +}); + +describe("resolveTargets", () => { + it("throws when no -m and no SCRAPE_TARGETS", () => { + expect(() => resolveTargets(undefined)).toThrow(/No model targets/); + }); + + it("parses, validates, and dedupes repeatable -m values", () => { + process.env.BRIGHTDATA_API_TOKEN = "test-token"; + const resolved = resolveTargets([ + "chatgpt:brightdata:online", + "chatgpt:brightdata:online", + "google-ai-mode:brightdata:online", + ]); + expect(resolved.map((r) => r.label)).toEqual(["chatgpt:brightdata:online", "google-ai-mode:brightdata:online"]); + expect(resolved[0].config.model).toBe("chatgpt"); + expect(resolved[0].config.webSearch).toBe(true); + }); + + it("falls back to SCRAPE_TARGETS when no -m is given", () => { + process.env.BRIGHTDATA_API_TOKEN = "test-token"; + process.env.SCRAPE_TARGETS = "chatgpt:brightdata:online"; + expect(resolveTargets(undefined).map((r) => r.label)).toEqual(["chatgpt:brightdata:online"]); + }); + + it("errors clearly when the provider is not configured", () => { + expect(() => resolveTargets(["chatgpt:brightdata:online"])).toThrow(/not configured/); + }); +}); + +describe("applyResearchTarget", () => { + it("sets ONBOARDING_LLM_TARGET when a model is given", () => { + applyResearchTarget("claude:anthropic-api"); + expect(process.env.ONBOARDING_LLM_TARGET).toBe("claude:anthropic-api"); + }); + it("leaves it unset for empty input", () => { + applyResearchTarget(undefined); + expect(process.env.ONBOARDING_LLM_TARGET).toBeUndefined(); + }); +}); diff --git a/apps/cli/src/core/targets.ts b/apps/cli/src/core/targets.ts new file mode 100644 index 00000000..821c4a89 --- /dev/null +++ b/apps/cli/src/core/targets.ts @@ -0,0 +1,67 @@ +import { formatScrapeTarget, type ModelConfig, parseScrapeTargets } from "@workspace/config/scrape-targets"; +import { getProvider, type Provider } from "@workspace/lib/providers"; + +export interface ResolvedTarget { + config: ModelConfig; + provider: Provider; + /** Canonical `model:provider[:version][:online]` label. */ + label: string; +} + +/** + * Resolve the `-m/--model` targets for `eval`. + * + * `--model` is repeatable and each value may itself be a comma-separated list, + * so we flatten them all. When none are given we fall back to the deployment's + * `SCRAPE_TARGETS` (the same set the worker tracks on a schedule). + * + * Every target is validated up front — provider must exist, be configured + * (its API key is present), and accept the model — so we fail fast with a clear + * message instead of part-way through a long run. + */ +export function resolveTargets(models: string[] | undefined): ResolvedTarget[] { + const raw = models && models.length > 0 ? models.join(",") : process.env.SCRAPE_TARGETS; + if (!raw?.trim()) { + throw new Error( + "No model targets. Pass one or more with -m (e.g. -m chatgpt:brightdata:online) or set SCRAPE_TARGETS via `elmo init`.", + ); + } + + const configs = parseScrapeTargets(raw); + const seen = new Set(); + const resolved: ResolvedTarget[] = []; + + for (const config of configs) { + const label = formatScrapeTarget(config); + if (seen.has(label)) continue; + seen.add(label); + + const provider = getProvider(config.provider); + if (!provider.isConfigured()) { + throw new Error( + `Target "${label}" uses provider "${config.provider}", which is not configured (its API key is missing). Add it with \`elmo edit env\` or export it before running.`, + ); + } + const validationError = provider.validateTarget?.(config); + if (validationError) { + throw new Error(`Invalid target "${label}": ${validationError}`); + } + + resolved.push({ config, provider, label }); + } + + return resolved; +} + +/** + * `brainstorm` and `plan` run a single structured-research call, which only + * direct-API providers support. When the user passes `-m`, point the onboarding + * provider resolver at it via `ONBOARDING_LLM_TARGET` (the env override the + * resolver already honors, including its "must support structured research" + * validation). Returns the chosen provider id for display. + */ +export function applyResearchTarget(model: string | undefined): void { + if (model?.trim()) { + process.env.ONBOARDING_LLM_TARGET = model.trim(); + } +} diff --git a/apps/cli/src/core/ui.ts b/apps/cli/src/core/ui.ts new file mode 100644 index 00000000..4b45b3e8 --- /dev/null +++ b/apps/cli/src/core/ui.ts @@ -0,0 +1,55 @@ +import pc from "picocolors"; + +/** + * OSC 8 hyperlink: clickable in iTerm2, Windows Terminal, GNOME Terminal, etc. + * Falls back to plain text in unsupported terminals. + */ +export function link(text: string, url: string): string { + return `\x1b]8;;${url}\x07${text}\x1b]8;;\x07`; +} + +const ELMO_ASCII = [ + "", + " ▄▄ ", + " ██ ", + "▄█▀█▄ ██ ███▄███▄ ▄███▄ ", + "██▄█▀ ██ ██ ██ ██ ██ ██ ", + "▀█▄▄▄ ██ ██ ██ ██ ▀███▀ ", + "", +].join("\n"); + +export function printBanner(): void { + // text-blue-600 ≈ #2563EB → RGB(37, 99, 235) + const blue = "\x1b[38;2;37;99;235m"; + const reset = "\x1b[0m"; + console.log(`${blue}${ELMO_ASCII}${reset}`); +} + +/** + * Human-facing messages for the `lab` commands go to **stderr** so that the + * machine-readable result (CSV / markdown / JSON) printed to **stdout** stays + * clean for piping. Never log progress to stdout in a lab command. + */ +export const log = { + info: (msg: string) => process.stderr.write(`${pc.dim("›")} ${msg}\n`), + step: (msg: string) => process.stderr.write(`${pc.cyan("›")} ${msg}\n`), + warn: (msg: string) => process.stderr.write(`${pc.yellow("!")} ${msg}\n`), + error: (msg: string) => process.stderr.write(`${pc.red("✗")} ${msg}\n`), + success: (msg: string) => process.stderr.write(`${pc.green("✓")} ${msg}\n`), +}; + +/** + * Library code in `@workspace/lib` (onboarding, providers) logs progress via + * `console.log`/`console.info`, which write to **stdout**. The lab commands put + * their machine-readable result on stdout, so redirect those library logs to + * stderr to keep piped output clean. `console.error`/`console.warn` already go + * to stderr. + */ +export function routeLibraryLogsToStderr(): void { + const toErr = (...args: unknown[]) => { + process.stderr.write(`${args.map((a) => (typeof a === "string" ? a : String(a))).join(" ")}\n`); + }; + console.log = toErr as typeof console.log; + console.info = toErr as typeof console.info; + console.debug = toErr as typeof console.debug; +} diff --git a/apps/cli/src/core/version.ts b/apps/cli/src/core/version.ts new file mode 100644 index 00000000..d532ea05 --- /dev/null +++ b/apps/cli/src/core/version.ts @@ -0,0 +1,42 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import semver from "semver"; +import { log } from "./ui.js"; + +export async function getPackageVersion(): Promise { + const selfDir = path.dirname(fileURLToPath(import.meta.url)); + // dist/index.js sits one level under the package root; src/core/version.ts + // sits two. Walk up until we find the package.json with a version. + for (const rel of ["..", "../..", "../../.."]) { + try { + const contents = await fs.readFile(path.resolve(selfDir, rel, "package.json"), "utf8"); + const json = JSON.parse(contents) as { name?: string; version?: string }; + if (json.version && json.name === "@elmohq/cli") return json.version; + } catch { + // keep walking + } + } + return "0.0.0"; +} + +async function maybeNotifyNewVersion(currentVersion: string): Promise { + try { + const response = await fetch("https://registry.npmjs.org/@elmohq/cli/latest"); + if (!response.ok) return; + const data = (await response.json()) as { version?: string }; + if (!data.version) return; + if (semver.valid(currentVersion) && semver.lt(currentVersion, data.version)) { + log.warn(`New CLI version available (${data.version}). Run: npm install -g @elmohq/cli@latest`); + } + } catch { + // Ignore update errors + } +} + +/** Run `fn`, then surface a new-version notice once it settles. */ +export async function withVersionCheck(version: string, fn: () => Promise): Promise { + const notifyPromise = maybeNotifyNewVersion(version); + await fn(); + await notifyPromise.catch(() => undefined); +} diff --git a/apps/cli/src/deploy.ts b/apps/cli/src/deploy.ts new file mode 100644 index 00000000..80f29360 --- /dev/null +++ b/apps/cli/src/deploy.ts @@ -0,0 +1,1293 @@ +import { spawn, spawnSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import * as p from "@clack/prompts"; +import { formatScrapeTarget, parseScrapeTargets } from "@workspace/config/scrape-targets"; +import type { Command } from "commander"; +import { parse as parseDotenv } from "dotenv"; +import pc from "picocolors"; +import { link, printBanner } from "./core/ui.js"; +import { withVersionCheck } from "./core/version.js"; +import { submitNewsletterSignup, trackCliEvent } from "./telemetry.js"; + +// ── Types ──────────────────────────────────────────────────────────────────── + +type ComposeService = { + Service: string; + State: string; + Health?: string; + ExitCode?: number; +}; + +type InitOptions = { + dev?: boolean; + dir?: string; + dockerDir?: string; +}; + +type DirOption = { + dir?: string; +}; + +type PostgresMode = "docker" | "external"; + +type EnvMap = Record; + +// ── Constants ──────────────────────────────────────────────────────────────── + +const CONFIG_HOME = path.join(os.homedir(), ".elmo"); +const DEFAULT_APP_NAME = "Elmo"; +const DEFAULT_APP_ICON = "/icons/elmo-icon.svg"; +const DEFAULT_APP_PORT = 1515; +const LOCAL_DATABASE_URL = "postgres://postgres:postgres@postgres:5432/elmo"; +const TELEMETRY_DOC_URL = "https://elmohq.com/docs/developer-guide/telemetry"; + +// ── Logging ────────────────────────────────────────────────────────────────── + +const log = { + info: (msg: string) => p.log.info(msg), + warn: (msg: string) => p.log.warn(msg), + error: (msg: string) => p.log.error(msg), + success: (msg: string) => p.log.success(msg), + step: (msg: string) => p.log.step(msg), +}; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +function assertNotCancelled(value: T | symbol): asserts value is T { + if (p.isCancel(value)) { + p.cancel("Setup cancelled."); + process.exit(0); + } +} + +function generateSecret(bytes = 32): string { + return crypto.randomBytes(bytes).toString("base64url"); +} + +// ── Registration ───────────────────────────────────────────────────────────── + +export function registerDeployCommands(program: Command, version: string): void { + program + .command("init") + .description("set up local Elmo instance") + .option("--dev", "Use local build context (repo only)") + .option("--docker-dir ", "Path to Docker build context (dev mode)") + .action(async (_opts: object, cmd: Command) => { + await withVersionCheck(version, () => runInit(cmd.optsWithGlobals(), version)); + }); + + program + .command("compose") + .description("run Docker Compose commands using your Elmo config") + .allowUnknownOption(true) + .argument("[args...]", "Arguments passed to Docker Compose") + .action(async (args: string[], _opts: object, cmd: Command) => { + await withVersionCheck(version, () => runCompose(args, cmd.optsWithGlobals())); + }); + + program + .command("edit") + .description("change API keys, scrape targets, or the Docker Compose YAML") + .argument("", "which config file to edit") + .action(async (target: string, _opts: object, cmd: Command) => { + await runEdit(target, cmd.optsWithGlobals()); + }); +} + +// ── Command: init ──────────────────────────────────────────────────────────── + +async function runInit(options: InitOptions, version: string): Promise { + printBanner(); + p.intro(pc.bold("Setting up Elmo")); + + const cwd = process.cwd(); + + // ── Resolve config directory ───────────────────────────────────────── + const configDir = options.dir ? path.resolve(cwd, options.dir) : CONFIG_HOME; + + // ── .env safety check ──────────────────────────────────────────────── + const existingEnvPath = path.join(configDir, ".env"); + let preservedDeploymentId: string | undefined; + if (await fileExists(existingEnvPath)) { + const contents = await fs.readFile(existingEnvPath, "utf8"); + const isElmoEnv = contents.startsWith("# Rendered by elmo") || contents.startsWith("# Generated by elmo"); + + if (!isElmoEnv) { + p.log.warn(`A .env file already exists in ${configDir} and was NOT created by Elmo.`); + const overwrite = await p.confirm({ + message: "Overwrite the existing .env file? This cannot be undone.", + initialValue: false, + }); + assertNotCancelled(overwrite); + if (!overwrite) { + p.cancel("Setup cancelled. Choose a different directory with --dir."); + process.exit(0); + } + } else { + p.log.warn(`An existing Elmo config was found at ${configDir}.`); + const overwrite = await p.confirm({ + message: "Overwrite it with new values? Existing secrets (DATABASE_URL, API keys, etc.) will be replaced.", + initialValue: false, + }); + assertNotCancelled(overwrite); + if (!overwrite) { + p.cancel("Setup cancelled. Use `elmo edit env` to change individual values."); + process.exit(0); + } + preservedDeploymentId = parseDotenv(contents).DEPLOYMENT_ID; + } + } + + // ── Dev mode: resolve docker directory ─────────────────────────────── + let dockerDir: string | undefined; + let repoRoot: string; + + if (options.dev) { + if (options.dockerDir) { + dockerDir = path.resolve(cwd, options.dockerDir); + if (!(await fileExists(path.join(dockerDir, "Dockerfile")))) { + p.log.error(`Dockerfile not found in ${dockerDir}`); + process.exit(1); + } + } else { + dockerDir = await resolveDockerDirInteractive(cwd); + } + repoRoot = path.resolve(dockerDir, ".."); + } else { + repoRoot = cwd; + } + + // ── Data stores ────────────────────────────────────────────────────── + const postgresMode = await p.select({ + message: "PostgreSQL connection", + options: [ + { + value: "docker" as const, + label: "Run Postgres in Docker", + }, + { + value: "external" as const, + label: "Use existing Postgres (provide DATABASE_URL)", + }, + ], + initialValue: "docker" as PostgresMode, + }); + assertNotCancelled(postgresMode); + + const env: EnvMap = {}; + env.DEPLOYMENT_MODE = "local"; + env.VITE_DEPLOYMENT_MODE = "local"; + env.DEPLOYMENT_ID = preservedDeploymentId ?? crypto.randomUUID(); + env.BETTER_AUTH_SECRET = generateSecret(); + env.APP_NAME = DEFAULT_APP_NAME; + env.APP_ICON = DEFAULT_APP_ICON; + env.VITE_APP_NAME = DEFAULT_APP_NAME; + env.VITE_APP_ICON = DEFAULT_APP_ICON; + + if (postgresMode === "external") { + p.note("Must be an IPv4-compatible direct connection or database pooler.", "DATABASE_URL"); + const url = await p.password({ + message: "DATABASE_URL", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(url); + env.DATABASE_URL = url; + } else { + env.DATABASE_URL = LOCAL_DATABASE_URL; + } + + // ── AI providers ───────────────────────────────────────────────────── + const setupMode = await configureProvidersInteractive(env); + + // ── Telemetry ─────────────────────────────────────────────────────── + p.note( + [ + "Elmo is open source and maintained by a small team. Telemetry", + "from both the CLI and your local deployment (web + worker)", + "tells us things like which CLI versions are still in use, where", + "`elmo init` drops off, which providers people pick, and whether", + "new features actually get used. Without it we are flying blind", + "on what to fix or build next.", + "", + pc.bold("What we send:"), + " • deployment ID (random UUID stored as DEPLOYMENT_ID in your .env)", + " • CLI/app version, OS, arch, Node version, deployment mode", + " • command/event names + non-secret options (e.g. postgres mode)", + " • feature counts (prompts edited, brands created — never the names or text)", + " • IP address (recorded on each event by PostHog, used for geolocation)", + "", + pc.bold("What we never send:"), + " API keys, .env contents, brand names, prompt text, and scraped responses.", + "", + `Full breakdown: ${link(pc.cyan(TELEMETRY_DOC_URL), TELEMETRY_DOC_URL)}`, + "Toggle later by editing DISABLE_TELEMETRY in .env (`elmo edit env`).", + ].join("\n"), + "Telemetry", + ); + + const telemetryEnabled = await p.confirm({ + message: "Share telemetry?", + initialValue: true, + }); + assertNotCancelled(telemetryEnabled); + if (!telemetryEnabled) { + env.DISABLE_TELEMETRY = "1"; + } + + // ── Product updates ───────────────────────────────────────────────── + const updatesEmail = await p.text({ + message: "Enter your work email to receive product updates (optional)", + placeholder: "you@example.com", + }); + const email = p.isCancel(updatesEmail) ? undefined : updatesEmail || undefined; + + // ── Web app port ──────────────────────────────────────────────────── + const portInput = await p.text({ + message: "Web app port", + placeholder: String(DEFAULT_APP_PORT), + defaultValue: String(DEFAULT_APP_PORT), + validate: (v) => { + if (!v) return undefined; + const n = Number(v); + if (!Number.isInteger(n) || n < 1 || n > 65535) { + return "Must be an integer between 1 and 65535"; + } + return undefined; + }, + }); + assertNotCancelled(portInput); + const port = Number(portInput); + env.APP_URL = `http://localhost:${port}`; + env.VITE_APP_URL = env.APP_URL; + + // ── Write config ───────────────────────────────────────────────────── + const composeYaml = buildComposeYaml({ + dev: Boolean(options.dev), + postgresMode, + repoRoot, + dockerDir, + port, + version, + }); + + await ensureDir(configDir); + await writeConfigFiles(configDir, { + env, + composeYaml, + postgresMode, + dev: Boolean(options.dev), + version, + }); + + p.log.success(`Config written to ${configDir}`); + p.log.warn("Your generated .env file contains secrets — do not commit it to version control."); + + if (options.dev) { + p.log.info("Dev mode enabled. Run `elmo compose build` before starting."); + } + + const shouldStart = await p.confirm({ + message: "Start the stack now?", + initialValue: true, + }); + assertNotCancelled(shouldStart); + + if (shouldStart) { + await doStart(configDir); + } else { + p.log.info("You can start later with `elmo compose up -d`."); + } + + // CLI telemetry — silently dropped if the user opted out above. + await trackCliEvent(configDir, "cli_init", { + version, + os: process.platform, + arch: process.arch, + node_version: process.version, + postgres_mode: postgresMode, + dev_mode: Boolean(options.dev), + setup_mode: setupMode, + has_scraper: Boolean(env.BRIGHTDATA_API_TOKEN || env.OLOSTEP_API_KEY), + has_direct_api: hasDirectApiConfigured(env), + }); + + // Newsletter signup is a separate, explicit opt-in and runs even when + // telemetry is disabled. + if (email) { + await submitNewsletterSignup(configDir, email); + } + + p.log.message( + `If you find Elmo useful, star us on GitHub!\n ${link(pc.cyan("https://github.com/elmohq/elmo"), "https://github.com/elmohq/elmo")}`, + ); + + p.outro(pc.green("Setup complete!")); +} + +// ── Provider Configuration ─────────────────────────────────────────────────── + +const BRIGHTDATA_AFFILIATE = "https://get.brightdata.com/67h1b7h0shcn"; +const OLOSTEP_AFFILIATE = "https://olostep.com/?ref=elmo"; +const PROVIDERS_DOC_URL = "https://docs.elmohq.com/docs/user-guide/providers"; + +// Surfaces each scraper can track — the first two are the "recommended starter" set. +const BRIGHTDATA_MODELS = ["chatgpt", "google-ai-mode", "perplexity", "copilot", "gemini", "grok"] as const; + +const OLOSTEP_MODELS = [ + "chatgpt", + "google-ai-mode", + "google-ai-overview", + "perplexity", + "copilot", + "gemini", + "grok", +] as const; + +const DEFAULT_SCRAPER_MODELS = ["chatgpt", "google-ai-mode"] as const; + +const DEFAULT_OPENAI_MODEL = "gpt-5-mini"; +const DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-4-6"; +const DEFAULT_OPENROUTER_MODEL = "anthropic/claude-sonnet-4.6"; +const DEFAULT_MISTRAL_MODEL = "mistral-medium-latest"; + +async function configureProvidersInteractive(env: EnvMap): Promise<"recommended" | "custom"> { + p.note( + [ + "Elmo needs two kinds of providers:", + "", + pc.bold("1. A scraper") + " — to track ChatGPT and Google AI Mode (no public APIs):", + ` • ${pc.cyan("BrightData")} — cheap solid option, ~$0.45/mo per prompt`, + ` • ${pc.cyan("Olostep")} — premium option, powers Peec/AirOps, ~$2.25/mo per prompt`, + "", + pc.bold("2. A direct LLM API") + " — for low-latency tasks (onboarding analysis, sentiment scoring,", + " ad-hoc LLM calls). Required:", + ` • ${pc.cyan("OpenRouter")} — one key, all major models (recommended)`, + ` • ${pc.cyan("Anthropic / OpenAI / Mistral")} — direct provider keys`, + "", + "Pricing assumes Elmo's default cadence (5 runs/day × 2 surfaces).", + ].join("\n"), + "AI providers", + ); + + const mode = await p.select({ + message: "Setup mode", + options: [ + { value: "recommended" as const, label: "Recommended — one scraper + one direct API" }, + { value: "custom" as const, label: "Custom — pick each provider individually" }, + ], + initialValue: "recommended" as const, + }); + assertNotCancelled(mode); + + if (mode === "recommended") { + await configureProvidersRecommended(env); + } else { + await configureProvidersCustom(env); + } + return mode; +} + +async function configureProvidersRecommended(env: EnvMap): Promise { + const targets: string[] = []; + + // ── Scraper ───────────────────────────────────────────────────────────── + const scraper = await p.select({ + message: "Scraper (tracks ChatGPT + Google AI Mode)", + options: [ + { value: "brightdata" as const, label: "BrightData — ~$0.45/mo per prompt (cheaper)" }, + { value: "olostep" as const, label: "Olostep — ~$2.25/mo per prompt (premium)" }, + ], + initialValue: "brightdata" as const, + }); + assertNotCancelled(scraper); + await collectScraperKey(scraper, env); + for (const model of DEFAULT_SCRAPER_MODELS) { + targets.push(formatScrapeTarget({ model, provider: scraper, webSearch: true })); + } + + // ── Direct API ────────────────────────────────────────────────────────── + const direct = await p.select({ + message: "Direct LLM API (powers onboarding analysis + sentiment scoring)", + options: [ + { value: "openrouter" as const, label: "OpenRouter — one key, all major models (recommended)" }, + { value: "anthropic" as const, label: "Anthropic — direct Claude" }, + { value: "openai" as const, label: "OpenAI — direct GPT-* models" }, + { value: "mistral" as const, label: "Mistral — direct Mistral models" }, + ], + initialValue: "openrouter" as const, + }); + assertNotCancelled(direct); + await collectDirectApiQuick(direct, env); + + await finalizeScrapeTargets(env, targets, { skipEdit: true }); +} + +async function configureProvidersCustom(env: EnvMap): Promise { + const targets: string[] = []; + + p.log.step(pc.bold("Step 1 of 2 — Direct LLM API (at least one is required)")); + // Order matches the auto-pick preference in onboarding/llm.ts so the first + // provider asked is the one onboarding will reach for by default. + while (!hasDirectApiConfigured(env)) { + await collectOpenRouter(env, targets); + await collectAnthropic(env, targets); + await collectOpenAI(env, targets); + await collectMistral(env, targets); + if (!hasDirectApiConfigured(env)) { + p.log.warn( + "Onboarding analysis and other low-latency LLM tasks require a direct API. Configure at least one before continuing.", + ); + } + } + + p.log.step(pc.bold("Step 2 of 2 — Scrapers (optional, but needed to track ChatGPT / Google AI Mode)")); + await collectBrightData(env, targets); + await collectOlostep(env, targets); + await collectDataForSEO(env, targets); + + await finalizeScrapeTargets(env, targets); +} + +function hasDirectApiConfigured(env: EnvMap): boolean { + return Boolean(env.ANTHROPIC_API_KEY || env.OPENAI_API_KEY || env.MISTRAL_API_KEY || env.OPENROUTER_API_KEY); +} + +async function collectScraperKey(scraper: "brightdata" | "olostep", env: EnvMap): Promise { + if (scraper === "brightdata") { + p.log.info(`Sign up: ${link(pc.cyan(BRIGHTDATA_AFFILIATE), BRIGHTDATA_AFFILIATE)}`); + const key = await p.password({ + message: "BrightData API token", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.BRIGHTDATA_API_TOKEN = key; + } else { + p.log.info(`Sign up: ${link(pc.cyan(OLOSTEP_AFFILIATE), OLOSTEP_AFFILIATE)}`); + const key = await p.password({ + message: "Olostep API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.OLOSTEP_API_KEY = key; + } +} + +async function collectDirectApiQuick( + kind: "openrouter" | "anthropic" | "openai" | "mistral", + env: EnvMap, +): Promise { + if (kind === "openrouter") { + const key = await p.password({ + message: "OpenRouter API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.OPENROUTER_API_KEY = key; + } else if (kind === "anthropic") { + const key = await p.password({ + message: "Anthropic API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.ANTHROPIC_API_KEY = key; + } else if (kind === "openai") { + const key = await p.password({ + message: "OpenAI API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.OPENAI_API_KEY = key; + } else { + const key = await p.password({ + message: "Mistral API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.MISTRAL_API_KEY = key; + } +} + +async function collectBrightData(env: EnvMap, targets: string[]): Promise { + const enable = await p.confirm({ + message: `Configure ${pc.bold("BrightData")}? (~$0.45/mo per prompt)`, + initialValue: true, + }); + assertNotCancelled(enable); + if (!enable) return; + + p.log.info(`Sign up and generate an API token: ${link(pc.cyan(BRIGHTDATA_AFFILIATE), BRIGHTDATA_AFFILIATE)}`); + const key = await p.password({ + message: "BrightData API token", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.BRIGHTDATA_API_TOKEN = key; + + await pickScraperTargets({ + providerLabel: "BrightData", + providerId: "brightdata", + allModels: BRIGHTDATA_MODELS as readonly string[], + targets, + }); +} + +async function collectOlostep(env: EnvMap, targets: string[]): Promise { + const enable = await p.confirm({ + message: `Configure ${pc.bold("Olostep")}? (~$2.25/mo per prompt)`, + initialValue: false, + }); + assertNotCancelled(enable); + if (!enable) return; + + p.log.info(`Grab an API key: ${link(pc.cyan(OLOSTEP_AFFILIATE), OLOSTEP_AFFILIATE)}`); + const key = await p.password({ + message: "Olostep API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.OLOSTEP_API_KEY = key; + + await pickScraperTargets({ + providerLabel: "Olostep", + providerId: "olostep", + allModels: OLOSTEP_MODELS as readonly string[], + targets, + }); +} + +async function pickScraperTargets(args: { + providerLabel: string; + providerId: "brightdata" | "olostep"; + allModels: readonly string[]; + targets: string[]; +}): Promise { + const selected = (await p.multiselect({ + message: `LLM Providers to track via ${args.providerLabel}`, + options: args.allModels.map((model) => ({ value: model, label: model })), + required: true, + initialValues: [...DEFAULT_SCRAPER_MODELS], + })) as string[] | symbol; + assertNotCancelled(selected); + + for (const model of selected) { + args.targets.push(formatScrapeTarget({ model, provider: args.providerId, webSearch: true })); + } +} + +async function collectAnthropic(env: EnvMap, targets: string[]): Promise { + const enable = await p.confirm({ + message: `Configure ${pc.bold("Anthropic API")}? (direct Claude — ~$4–5/mo per prompt per model)`, + initialValue: false, + }); + assertNotCancelled(enable); + if (!enable) return; + + const key = await p.password({ + message: "Anthropic API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.ANTHROPIC_API_KEY = key; + + const model = await p.text({ + message: "Claude model", + placeholder: DEFAULT_ANTHROPIC_MODEL, + defaultValue: DEFAULT_ANTHROPIC_MODEL, + }); + assertNotCancelled(model); + const slug = model || DEFAULT_ANTHROPIC_MODEL; + + const webSearch = await p.confirm({ + message: "Enable web search? (recommended, but more expensive)", + initialValue: true, + }); + assertNotCancelled(webSearch); + + targets.push(formatScrapeTarget({ model: "claude", provider: "anthropic-api", version: slug, webSearch })); +} + +async function collectOpenAI(env: EnvMap, targets: string[]): Promise { + const enable = await p.confirm({ + message: `Configure ${pc.bold("OpenAI API")}? (gpt-* with web search — not the real ChatGPT UI)`, + initialValue: false, + }); + assertNotCancelled(enable); + if (!enable) return; + + const key = await p.password({ + message: "OpenAI API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.OPENAI_API_KEY = key; + + const model = await p.text({ + message: "OpenAI model", + placeholder: DEFAULT_OPENAI_MODEL, + defaultValue: DEFAULT_OPENAI_MODEL, + }); + assertNotCancelled(model); + const slug = model || DEFAULT_OPENAI_MODEL; + + const webSearch = await p.confirm({ + message: "Enable web search? (recommended, but more expensive)", + initialValue: true, + }); + assertNotCancelled(webSearch); + + targets.push(formatScrapeTarget({ model: "chatgpt", provider: "openai-api", version: slug, webSearch })); +} + +async function collectMistral(env: EnvMap, targets: string[]): Promise { + const enable = await p.confirm({ + message: `Configure ${pc.bold("Mistral API")}? (direct Mistral models)`, + initialValue: false, + }); + assertNotCancelled(enable); + if (!enable) return; + + const key = await p.password({ + message: "Mistral API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.MISTRAL_API_KEY = key; + + const model = await p.text({ + message: "Mistral model", + placeholder: DEFAULT_MISTRAL_MODEL, + defaultValue: DEFAULT_MISTRAL_MODEL, + }); + assertNotCancelled(model); + const slug = model || DEFAULT_MISTRAL_MODEL; + + const webSearch = await p.confirm({ + message: "Enable web search? (recommended, but more expensive)", + initialValue: true, + }); + assertNotCancelled(webSearch); + + targets.push(formatScrapeTarget({ model: "mistral", provider: "mistral-api", version: slug, webSearch })); +} + +async function collectOpenRouter(env: EnvMap, targets: string[]): Promise { + const enable = await p.confirm({ + message: `Configure ${pc.bold("OpenRouter")}? (one key, many hosted models)`, + initialValue: false, + }); + assertNotCancelled(enable); + if (!enable) return; + + const key = await p.password({ + message: "OpenRouter API key", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(key); + env.OPENROUTER_API_KEY = key; + + const model = await p.text({ + message: "OpenRouter model slug", + placeholder: DEFAULT_OPENROUTER_MODEL, + defaultValue: DEFAULT_OPENROUTER_MODEL, + }); + assertNotCancelled(model); + const slug = model || DEFAULT_OPENROUTER_MODEL; + + const webSearch = await p.confirm({ + message: "Enable web search? (recommended, but more expensive)", + initialValue: true, + }); + assertNotCancelled(webSearch); + + targets.push(formatScrapeTarget({ model: "claude", provider: "openrouter", version: slug, webSearch })); +} + +async function collectDataForSEO(env: EnvMap, targets: string[]): Promise { + const enable = await p.confirm({ + message: `Configure ${pc.bold("DataForSEO")}? (Google AI Mode scraping)`, + initialValue: false, + }); + assertNotCancelled(enable); + if (!enable) return; + + const login = await p.text({ + message: "DataForSEO login", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(login); + env.DATAFORSEO_LOGIN = login; + + const pwd = await p.password({ + message: "DataForSEO password", + validate: (v) => (!v ? "Required" : undefined), + }); + assertNotCancelled(pwd); + env.DATAFORSEO_PASSWORD = pwd; + + const addTarget = await p.confirm({ + message: "Also scrape Google AI Mode via DataForSEO? (google-ai-mode:dataforseo:online)", + initialValue: false, + }); + assertNotCancelled(addTarget); + if (addTarget) { + targets.push(formatScrapeTarget({ model: "google-ai-mode", provider: "dataforseo", webSearch: true })); + } +} + +async function finalizeScrapeTargets( + env: EnvMap, + targets: string[], + options: { skipEdit?: boolean } = {}, +): Promise { + const deduped = dedupeTargets(targets); + + if (!deduped) { + p.log.warn("No SCRAPE_TARGETS configured. Elmo will not run scheduled checks until you set them."); + p.log.info(`Reference: ${link(pc.cyan(PROVIDERS_DOC_URL), PROVIDERS_DOC_URL)}`); + + const addManual = await p.confirm({ + message: "Enter SCRAPE_TARGETS manually now?", + initialValue: false, + }); + assertNotCancelled(addManual); + if (addManual) { + const manual = await p.text({ + message: "SCRAPE_TARGETS (model:provider[:version][:online], comma-separated)", + placeholder: "chatgpt:brightdata:online,google-ai-mode:brightdata:online", + validate: validateScrapeTargetsInput, + }); + assertNotCancelled(manual); + env.SCRAPE_TARGETS = manual; + } + return; + } + + if (options.skipEdit) { + env.SCRAPE_TARGETS = deduped; + return; + } + + const customize = await p.confirm({ + message: "Edit SCRAPE_TARGETS before saving?", + initialValue: false, + }); + assertNotCancelled(customize); + + if (customize) { + p.log.info(`Reference: ${link(pc.cyan(PROVIDERS_DOC_URL), PROVIDERS_DOC_URL)}`); + const manual = await p.text({ + message: "SCRAPE_TARGETS", + initialValue: deduped, + validate: validateScrapeTargetsInput, + }); + assertNotCancelled(manual); + env.SCRAPE_TARGETS = manual; + p.log.step(`SCRAPE_TARGETS:\n ${pc.cyan(manual)}`); + } else { + env.SCRAPE_TARGETS = deduped; + } +} + +function validateScrapeTargetsInput(value: string | undefined): string | undefined { + if (!value) return "Required"; + try { + parseScrapeTargets(value); + } catch (error) { + return error instanceof Error ? error.message.split("\n")[0] : String(error); + } + return undefined; +} + +function dedupeTargets(targets: string[]): string { + const seen = new Set(); + const out: string[] = []; + for (const t of targets) { + if (seen.has(t)) continue; + seen.add(t); + out.push(t); + } + return out.join(","); +} + +// ── Start helper (used by init) ────────────────────────────────────────────── + +async function doStart(configDir: string): Promise { + assertDockerRunning(); + + log.step("Starting Docker Compose stack..."); + await runDockerCompose(configDir, ["up", "-d"]); + + const s = p.spinner(); + s.start("Waiting for services to become healthy..."); + const ok = await waitForHealthy(configDir, 180_000); + if (ok) { + s.stop("All services healthy!"); + } else { + s.stop("Health check timed out."); + p.log.warn("Some services did not report healthy status."); + } + + log.info("Examples:"); + console.log(` ${pc.bold("elmo compose logs -f")}`); + console.log(` ${pc.bold("elmo compose logs -f web")}`); + console.log(` ${pc.bold("elmo compose ps")}`); + console.log(` ${pc.bold("elmo compose down")}`); +} + +// ── Command: compose ───────────────────────────────────────────────────────── + +async function runCompose(args: string[], options: DirOption): Promise { + const configDir = await resolveConfigDir(options.dir); + assertDockerRunning(); + await runDockerCompose(configDir, args); +} + +// ── Command: edit ──────────────────────────────────────────────────────────── + +async function runEdit(target: string, options: DirOption): Promise { + const configDir = await resolveConfigDir(options.dir); + + let filePath: string; + if (target === "env") { + filePath = path.join(configDir, ".env"); + } else if (target === "compose") { + filePath = path.join(configDir, "elmo.yaml"); + } else { + throw new Error(`Unknown edit target: ${target}. Use \`env\` or \`compose\`.`); + } + + if (!(await fileExists(filePath))) { + throw new Error(`File not found: ${filePath}`); + } + + const editorEnv = process.env.VISUAL || process.env.EDITOR || "nano"; + const parts = editorEnv.split(/\s+/).filter(Boolean); + const cmd = parts[0] ?? "nano"; + const args = [...parts.slice(1), filePath]; + + await new Promise((resolve, reject) => { + const child = spawn(cmd, args, { stdio: "inherit" }); + child.on("close", (code) => { + if (code === 0) resolve(); + else reject(new Error(`${cmd} exited with code ${code}`)); + }); + child.on("error", (err) => reject(err)); + }); + + log.info("Restart the stack with `elmo compose up -d` to apply changes."); +} + +// ── Compose YAML Builder ───────────────────────────────────────────────────── + +function buildComposeYaml(options: { + dev: boolean; + postgresMode: PostgresMode; + repoRoot: string; + dockerDir?: string; + port: number; + version: string; +}): string { + const services: string[] = []; + const volumes = new Set(); + + const dependsOnWeb: string[] = []; + const dependsOnWorker: string[] = []; + + const dependencyConditions: Record = { + postgres: "service_healthy", + "db-migrate": "service_completed_successfully", + }; + + const dockerfilePath = options.dockerDir + ? path.relative(options.repoRoot, path.join(options.dockerDir, "Dockerfile")) + : "docker/Dockerfile"; + + if (options.postgresMode === "docker") { + services.push(buildPostgresService()); + services.push( + buildDbMigrateService({ + dev: options.dev, + dockerfilePath, + repoRoot: options.repoRoot, + version: options.version, + }), + ); + dependsOnWeb.push("db-migrate"); + dependsOnWorker.push("db-migrate"); + volumes.add("postgres_data"); + } + + services.push( + buildWebService({ + dev: options.dev, + dependsOn: dependsOnWeb, + dependencyConditions, + repoRoot: options.repoRoot, + dockerfilePath, + port: options.port, + version: options.version, + }), + ); + services.push( + buildWorkerService({ + dev: options.dev, + dependsOn: dependsOnWorker, + dependencyConditions, + repoRoot: options.repoRoot, + dockerfilePath, + version: options.version, + }), + ); + + const lines = [renderedByHeader(options.version), "", "name: elmo", "", "services:"]; + lines.push(...services.map((service) => indentBlock(service, 2))); + + if (volumes.size > 0) { + lines.push("", "volumes:"); + for (const volume of volumes) { + lines.push(` ${volume}:`); + } + } + + return `${lines.join("\n")}\n`; +} + +function buildPostgresService(): string { + return [ + "postgres:", + " image: postgres:16-alpine", + " environment:", + " POSTGRES_USER: postgres", + " POSTGRES_PASSWORD: postgres", + " POSTGRES_DB: elmo", + " volumes:", + " - postgres_data:/var/lib/postgresql/data", + " ports:", + ' - "5432:5432"', + " healthcheck:", + ' test: ["CMD-SHELL", "pg_isready -U postgres"]', + " interval: 5s", + " timeout: 5s", + " retries: 5", + " start_period: 30s", + ].join("\n"); +} + +function buildDbMigrateService(options: { + dev: boolean; + dockerfilePath: string; + repoRoot: string; + version: string; +}): string { + const lines = ["db-migrate:"]; + if (options.dev) { + lines.push( + " build:", + ` context: ${options.repoRoot}`, + ` dockerfile: ${options.dockerfilePath}`, + " target: migrate", + ); + } else { + lines.push(` image: elmohq/elmo-db-migrate:${options.version}`); + } + + lines.push( + " environment:", + " - DATABASE_URL=postgres://postgres:postgres@postgres:5432/elmo", + " depends_on:", + " postgres:", + " condition: service_healthy", + ); + + return lines.join("\n"); +} + +function buildWebService(options: { + dev: boolean; + dependsOn: string[]; + dependencyConditions: Record; + repoRoot: string; + dockerfilePath: string; + port: number; + version: string; +}): string { + const lines = ["web:"]; + if (options.dev) { + lines.push( + " build:", + ` context: ${options.repoRoot}`, + ` dockerfile: ${options.dockerfilePath}`, + " target: web", + " args:", + " DEPLOYMENT_MODE: local", + ); + } else { + lines.push(` image: elmohq/elmo-web:${options.version}`); + } + + lines.push(" env_file:", " - path: .env", " required: true", " ports:", ` - "${options.port}:3000"`); + + if (options.dependsOn.length > 0) { + lines.push(" depends_on:"); + for (const service of options.dependsOn) { + const condition = options.dependencyConditions[service] ?? "service_started"; + lines.push(` ${service}:`, ` condition: ${condition}`); + } + } + + return lines.join("\n"); +} + +function buildWorkerService(options: { + dev: boolean; + dependsOn: string[]; + dependencyConditions: Record; + repoRoot: string; + dockerfilePath: string; + version: string; +}): string { + const lines = ["worker:"]; + if (options.dev) { + lines.push( + " build:", + ` context: ${options.repoRoot}`, + ` dockerfile: ${options.dockerfilePath}`, + " target: worker", + " args:", + " DEPLOYMENT_MODE: local", + ); + } else { + lines.push(` image: elmohq/elmo-worker:${options.version}`); + } + + lines.push(" env_file:", " - path: .env", " required: true"); + + if (options.dependsOn.length > 0) { + lines.push(" depends_on:"); + for (const service of options.dependsOn) { + const condition = options.dependencyConditions[service] ?? "service_started"; + lines.push(` ${service}:`, ` condition: ${condition}`); + } + } + + return lines.join("\n"); +} + +function indentBlock(block: string, spaces: number): string { + const indent = " ".repeat(spaces); + return block + .split("\n") + .map((line) => `${indent}${line}`) + .join("\n"); +} + +// ── Docker Helpers ─────────────────────────────────────────────────────────── + +async function getComposeServices(configDir: string): Promise { + const output = await runDockerComposeCapture(configDir, ["ps", "--format", "json"]); + if (!output.trim()) { + return []; + } + try { + const trimmed = output.trim(); + const parsed = JSON.parse(trimmed); + if (Array.isArray(parsed)) { + return parsed as ComposeService[]; + } + if (typeof parsed === "object" && parsed !== null) { + return [parsed as ComposeService]; + } + return []; + } catch { + try { + return output + .trim() + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line) as ComposeService); + } catch { + log.warn("Unable to parse docker compose status."); + return []; + } + } +} + +function isServiceReady(service: ComposeService): boolean { + if (service.Health) { + return service.Health === "healthy"; + } + if (service.State?.startsWith("running")) { + return true; + } + return false; +} + +async function waitForHealthy(configDir: string, timeoutMs: number): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const services = await getComposeServices(configDir); + if (services.length > 0 && services.every(isServiceReady)) { + return true; + } + await sleep(3000); + } + return false; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function runDockerCompose(configDir: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const composeFile = path.join(configDir, "elmo.yaml"); + const commandArgs = ["compose", "-f", composeFile, ...args]; + const child = spawn("docker", commandArgs, { + stdio: "inherit", + }); + child.on("close", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`docker compose exited with code ${code}`)); + } + }); + }); +} + +function runDockerComposeCapture(configDir: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const composeFile = path.join(configDir, "elmo.yaml"); + const commandArgs = ["compose", "-f", composeFile, ...args]; + const child = spawn("docker", commandArgs); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (data: Buffer) => { + stdout += data.toString(); + }); + child.stderr.on("data", (data: Buffer) => { + stderr += data.toString(); + }); + child.on("close", (code) => { + if (code === 0) { + resolve(stdout); + } else { + reject(new Error(stderr || `docker compose exited with code ${code}`)); + } + }); + }); +} + +function assertDockerRunning(): void { + const result = spawnSync("docker", ["info"], { + stdio: "ignore", + }); + if (result.status !== 0) { + throw new Error("Docker does not appear to be running. Start Docker and try again."); + } +} + +// ── Docker Dir Resolution ──────────────────────────────────────────────────── + +async function resolveDockerDirInteractive(cwd: string): Promise { + const inCwd = await fileExists(path.join(cwd, "Dockerfile")); + const inDockerDir = await fileExists(path.join(cwd, "docker", "Dockerfile")); + const defaultDir = inCwd ? "." : inDockerDir ? "docker" : "."; + + const dir = await p.text({ + message: "Path to docker directory (contains Dockerfile)", + defaultValue: defaultDir, + }); + assertNotCancelled(dir); + + const resolved = path.resolve(cwd, dir); + if (!(await fileExists(path.join(resolved, "Dockerfile")))) { + p.log.error(`Dockerfile not found in ${resolved}. Provide the directory that contains Dockerfile.`); + process.exit(1); + } + + return resolved; +} + +// ── Config Dir Resolution ──────────────────────────────────────────────────── + +async function resolveConfigDir(explicitDir?: string): Promise { + const resolved = explicitDir ? path.resolve(process.cwd(), explicitDir) : CONFIG_HOME; + const composePath = path.join(resolved, "elmo.yaml"); + if (!(await fileExists(composePath))) { + if (explicitDir) { + throw new Error( + `Config directory does not contain elmo.yaml: ${resolved}\nRun \`elmo init --dir ${explicitDir}\` to create it.`, + ); + } + throw new Error(`No config found at ${resolved}. Run \`elmo init\` to create one, or specify --dir.`); + } + return resolved; +} + +// ── File & Config Helpers ──────────────────────────────────────────────────── + +async function writeConfigFiles( + configDir: string, + initConfig: { + env: EnvMap; + composeYaml: string; + postgresMode: PostgresMode; + dev: boolean; + version: string; + }, +): Promise { + const envPath = path.join(configDir, ".env"); + const composePath = path.join(configDir, "elmo.yaml"); + + await ensureDir(configDir); + await fs.writeFile(envPath, buildEnvFile(initConfig.env, initConfig.version), "utf8"); + await fs.writeFile(composePath, initConfig.composeYaml, "utf8"); +} + +function renderedByHeader(version: string): string { + return [ + `# Rendered by elmo ${version} on ${new Date().toISOString()}`, + "# Re-run `elmo init` after upgrading the CLI to refresh this file.", + ].join("\n"); +} + +function buildEnvFile(env: EnvMap, version: string): string { + const lines = [renderedByHeader(version), "# WARNING: contains secrets. Do not commit.", ""]; + + for (const [key, rawValue] of Object.entries(env)) { + if (rawValue === undefined) { + continue; + } + lines.push(`${key}=${formatEnvValue(rawValue)}`); + } + + return `${lines.join("\n")}\n`; +} + +function formatEnvValue(value: string): string { + if (value === "") { + return '""'; + } + if (/[\s#"']/u.test(value)) { + const escaped = value.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + return `"${escaped}"`; + } + return value; +} + +async function fileExists(target: string): Promise { + try { + await fs.access(target); + return true; + } catch { + return false; + } +} + +async function ensureDir(dir: string): Promise { + await fs.mkdir(dir, { recursive: true }); +} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 54c168fd..f892b950 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -1,99 +1,12 @@ #!/usr/bin/env node -import { spawn, spawnSync } from "node:child_process"; -import crypto from "node:crypto"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import * as p from "@clack/prompts"; -import { formatScrapeTarget, parseScrapeTargets } from "@workspace/config/scrape-targets"; import { Command } from "commander"; -import { parse as parseDotenv } from "dotenv"; import pc from "picocolors"; -import semver from "semver"; -import { submitNewsletterSignup, trackCliEvent } from "./telemetry.js"; - -// ── Types ──────────────────────────────────────────────────────────────────── - -type ComposeService = { - Service: string; - State: string; - Health?: string; - ExitCode?: number; -}; - -type InitOptions = { - dev?: boolean; - dir?: string; - dockerDir?: string; -}; - -type DirOption = { - dir?: string; -}; - -type PostgresMode = "docker" | "external"; - -type EnvMap = Record; - -// ── Constants ──────────────────────────────────────────────────────────────── - -const CONFIG_HOME = path.join(os.homedir(), ".elmo"); -const DEFAULT_APP_NAME = "Elmo"; -const DEFAULT_APP_ICON = "/icons/elmo-icon.svg"; -const DEFAULT_APP_PORT = 1515; -const LOCAL_DATABASE_URL = "postgres://postgres:postgres@postgres:5432/elmo"; -const TELEMETRY_DOC_URL = "https://elmohq.com/docs/developer-guide/telemetry"; - -// ── Banner ─────────────────────────────────────────────────────────────────── - -const ELMO_ASCII = [ - "", - " ▄▄ ", - " ██ ", - "▄█▀█▄ ██ ███▄███▄ ▄███▄ ", - "██▄█▀ ██ ██ ██ ██ ██ ██ ", - "▀█▄▄▄ ██ ██ ██ ██ ▀███▀ ", - "", -].join("\n"); - -function printBanner(): void { - // text-blue-600 ≈ #2563EB → RGB(37, 99, 235) - const blue = "\x1b[38;2;37;99;235m"; - const reset = "\x1b[0m"; - console.log(`${blue}${ELMO_ASCII}${reset}`); -} - -// ── Logging ────────────────────────────────────────────────────────────────── - -const log = { - info: (msg: string) => p.log.info(msg), - warn: (msg: string) => p.log.warn(msg), - error: (msg: string) => p.log.error(msg), - success: (msg: string) => p.log.success(msg), - step: (msg: string) => p.log.step(msg), -}; - -// ── Helpers ────────────────────────────────────────────────────────────────── - -function assertNotCancelled(value: T | symbol): asserts value is T { - if (p.isCancel(value)) { - p.cancel("Setup cancelled."); - process.exit(0); - } -} - -function generateSecret(bytes = 32): string { - return crypto.randomBytes(bytes).toString("base64url"); -} - -function link(text: string, url: string): string { - // OSC 8 hyperlink: clickable in iTerm2, Windows Terminal, GNOME Terminal, etc. - // Falls back to plain text in unsupported terminals. - return `\x1b]8;;${url}\x07${text}\x1b]8;;\x07`; -} - -// ── Main ───────────────────────────────────────────────────────────────────── +import { registerBrainstorm } from "./commands/brainstorm.js"; +import { registerEval } from "./commands/eval.js"; +import { registerPlan } from "./commands/plan.js"; +import { printBanner } from "./core/ui.js"; +import { getPackageVersion } from "./core/version.js"; +import { registerDeployCommands } from "./deploy.js"; async function main() { const version = await getPackageVersion(); @@ -109,1288 +22,24 @@ async function main() { program.outputHelp(); }); - program - .command("init") - .description("set up local Elmo instance") - .option("--dev", "Use local build context (repo only)") - .option("--docker-dir ", "Path to Docker build context (dev mode)") - .action(async (_opts: object, cmd: Command) => { - await withVersionCheck(version, () => runInit(cmd.optsWithGlobals(), version)); - }); + // ── Deploy: stand up / manage a self-hosted instance ───────────────────── + registerDeployCommands(program, version); - program - .command("compose") - .description("run Docker Compose commands using your Elmo config") - .allowUnknownOption(true) - .argument("[args...]", "Arguments passed to Docker Compose") - .action(async (args: string[], _opts: object, cmd: Command) => { - await withVersionCheck(version, () => runCompose(args, cmd.optsWithGlobals())); - }); - - program - .command("edit") - .description("change API keys, scrape targets, or the Docker Compose YAML") - .argument("", "which config file to edit") - .action(async (target: string, _opts: object, cmd: Command) => { - await runEdit(target, cmd.optsWithGlobals()); + // ── Lab: one-off AEO runs against your configured providers ────────────── + // Registered in the order you'd actually run them: brainstorm → eval → plan. + const lab = program + .command("lab") + .description("one-off AEO runs (brainstorm prompts, eval them, plan improvements)") + .action(() => { + lab.outputHelp(); }); + registerBrainstorm(lab); + registerEval(lab); + registerPlan(lab); await program.parseAsync(process.argv); } -async function withVersionCheck(version: string, fn: () => Promise): Promise { - const notifyPromise = maybeNotifyNewVersion(version); - await fn(); - await notifyPromise.catch(() => undefined); -} - -// ── Command: init ──────────────────────────────────────────────────────────── - -async function runInit(options: InitOptions, version: string): Promise { - printBanner(); - p.intro(pc.bold("Setting up Elmo")); - - const cwd = process.cwd(); - - // ── Resolve config directory ───────────────────────────────────────── - const configDir = options.dir ? path.resolve(cwd, options.dir) : CONFIG_HOME; - - // ── .env safety check ──────────────────────────────────────────────── - const existingEnvPath = path.join(configDir, ".env"); - let preservedDeploymentId: string | undefined; - if (await fileExists(existingEnvPath)) { - const contents = await fs.readFile(existingEnvPath, "utf8"); - const isElmoEnv = contents.startsWith("# Rendered by elmo") || contents.startsWith("# Generated by elmo"); - - if (!isElmoEnv) { - p.log.warn(`A .env file already exists in ${configDir} and was NOT created by Elmo.`); - const overwrite = await p.confirm({ - message: "Overwrite the existing .env file? This cannot be undone.", - initialValue: false, - }); - assertNotCancelled(overwrite); - if (!overwrite) { - p.cancel("Setup cancelled. Choose a different directory with --dir."); - process.exit(0); - } - } else { - p.log.warn(`An existing Elmo config was found at ${configDir}.`); - const overwrite = await p.confirm({ - message: "Overwrite it with new values? Existing secrets (DATABASE_URL, API keys, etc.) will be replaced.", - initialValue: false, - }); - assertNotCancelled(overwrite); - if (!overwrite) { - p.cancel("Setup cancelled. Use `elmo edit env` to change individual values."); - process.exit(0); - } - preservedDeploymentId = parseDotenv(contents).DEPLOYMENT_ID; - } - } - - // ── Dev mode: resolve docker directory ─────────────────────────────── - let dockerDir: string | undefined; - let repoRoot: string; - - if (options.dev) { - if (options.dockerDir) { - dockerDir = path.resolve(cwd, options.dockerDir); - if (!(await fileExists(path.join(dockerDir, "Dockerfile")))) { - p.log.error(`Dockerfile not found in ${dockerDir}`); - process.exit(1); - } - } else { - dockerDir = await resolveDockerDirInteractive(cwd); - } - repoRoot = path.resolve(dockerDir, ".."); - } else { - repoRoot = cwd; - } - - // ── Data stores ────────────────────────────────────────────────────── - const postgresMode = await p.select({ - message: "PostgreSQL connection", - options: [ - { - value: "docker" as const, - label: "Run Postgres in Docker", - }, - { - value: "external" as const, - label: "Use existing Postgres (provide DATABASE_URL)", - }, - ], - initialValue: "docker" as PostgresMode, - }); - assertNotCancelled(postgresMode); - - const env: EnvMap = {}; - env.DEPLOYMENT_MODE = "local"; - env.VITE_DEPLOYMENT_MODE = "local"; - env.DEPLOYMENT_ID = preservedDeploymentId ?? crypto.randomUUID(); - env.BETTER_AUTH_SECRET = generateSecret(); - env.APP_NAME = DEFAULT_APP_NAME; - env.APP_ICON = DEFAULT_APP_ICON; - env.VITE_APP_NAME = DEFAULT_APP_NAME; - env.VITE_APP_ICON = DEFAULT_APP_ICON; - - if (postgresMode === "external") { - p.note("Must be an IPv4-compatible direct connection or database pooler.", "DATABASE_URL"); - const url = await p.password({ - message: "DATABASE_URL", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(url); - env.DATABASE_URL = url; - } else { - env.DATABASE_URL = LOCAL_DATABASE_URL; - } - - // ── AI providers ───────────────────────────────────────────────────── - const setupMode = await configureProvidersInteractive(env); - - // ── Telemetry ─────────────────────────────────────────────────────── - p.note( - [ - "Elmo is open source and maintained by a small team. Telemetry", - "from both the CLI and your local deployment (web + worker)", - "tells us things like which CLI versions are still in use, where", - "`elmo init` drops off, which providers people pick, and whether", - "new features actually get used. Without it we are flying blind", - "on what to fix or build next.", - "", - pc.bold("What we send:"), - " • deployment ID (random UUID stored as DEPLOYMENT_ID in your .env)", - " • CLI/app version, OS, arch, Node version, deployment mode", - " • command/event names + non-secret options (e.g. postgres mode)", - " • feature counts (prompts edited, brands created — never the names or text)", - " • IP address (recorded on each event by PostHog, used for geolocation)", - "", - pc.bold("What we never send:"), - " API keys, .env contents, brand names, prompt text, and scraped responses.", - "", - `Full breakdown: ${link(pc.cyan(TELEMETRY_DOC_URL), TELEMETRY_DOC_URL)}`, - "Toggle later by editing DISABLE_TELEMETRY in .env (`elmo edit env`).", - ].join("\n"), - "Telemetry", - ); - - const telemetryEnabled = await p.confirm({ - message: "Share telemetry?", - initialValue: true, - }); - assertNotCancelled(telemetryEnabled); - if (!telemetryEnabled) { - env.DISABLE_TELEMETRY = "1"; - } - - // ── Product updates ───────────────────────────────────────────────── - const updatesEmail = await p.text({ - message: "Enter your work email to receive product updates (optional)", - placeholder: "you@example.com", - }); - const email = p.isCancel(updatesEmail) ? undefined : updatesEmail || undefined; - - // ── Web app port ──────────────────────────────────────────────────── - const portInput = await p.text({ - message: "Web app port", - placeholder: String(DEFAULT_APP_PORT), - defaultValue: String(DEFAULT_APP_PORT), - validate: (v) => { - if (!v) return undefined; - const n = Number(v); - if (!Number.isInteger(n) || n < 1 || n > 65535) { - return "Must be an integer between 1 and 65535"; - } - return undefined; - }, - }); - assertNotCancelled(portInput); - const port = Number(portInput); - env.APP_URL = `http://localhost:${port}`; - env.VITE_APP_URL = env.APP_URL; - - // ── Write config ───────────────────────────────────────────────────── - const composeYaml = buildComposeYaml({ - dev: Boolean(options.dev), - postgresMode, - repoRoot, - dockerDir, - port, - version, - }); - - await ensureDir(configDir); - await writeConfigFiles(configDir, { - env, - composeYaml, - postgresMode, - dev: Boolean(options.dev), - version, - }); - - p.log.success(`Config written to ${configDir}`); - p.log.warn("Your generated .env file contains secrets — do not commit it to version control."); - - if (options.dev) { - p.log.info("Dev mode enabled. Run `elmo compose build` before starting."); - } - - const shouldStart = await p.confirm({ - message: "Start the stack now?", - initialValue: true, - }); - assertNotCancelled(shouldStart); - - if (shouldStart) { - await doStart(configDir); - } else { - p.log.info("You can start later with `elmo compose up -d`."); - } - - // CLI telemetry — silently dropped if the user opted out above. - await trackCliEvent(configDir, "cli_init", { - version, - os: process.platform, - arch: process.arch, - node_version: process.version, - postgres_mode: postgresMode, - dev_mode: Boolean(options.dev), - setup_mode: setupMode, - has_scraper: Boolean(env.BRIGHTDATA_API_TOKEN || env.OLOSTEP_API_KEY), - has_direct_api: hasDirectApiConfigured(env), - }); - - // Newsletter signup is a separate, explicit opt-in and runs even when - // telemetry is disabled. - if (email) { - await submitNewsletterSignup(configDir, email); - } - - p.log.message( - `If you find Elmo useful, star us on GitHub!\n ${link(pc.cyan("https://github.com/elmohq/elmo"), "https://github.com/elmohq/elmo")}`, - ); - - p.outro(pc.green("Setup complete!")); -} - -// ── Provider Configuration ─────────────────────────────────────────────────── - -const BRIGHTDATA_AFFILIATE = "https://get.brightdata.com/67h1b7h0shcn"; -const OLOSTEP_AFFILIATE = "https://olostep.com/?ref=elmo"; -const PROVIDERS_DOC_URL = "https://docs.elmohq.com/docs/user-guide/providers"; - -// Surfaces each scraper can track — the first two are the "recommended starter" set. -const BRIGHTDATA_MODELS = ["chatgpt", "google-ai-mode", "perplexity", "copilot", "gemini", "grok"] as const; - -const OLOSTEP_MODELS = [ - "chatgpt", - "google-ai-mode", - "google-ai-overview", - "perplexity", - "copilot", - "gemini", - "grok", -] as const; - -const DEFAULT_SCRAPER_MODELS = ["chatgpt", "google-ai-mode"] as const; - -const DEFAULT_OPENAI_MODEL = "gpt-5-mini"; -const DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-4-6"; -const DEFAULT_OPENROUTER_MODEL = "anthropic/claude-sonnet-4.6"; -const DEFAULT_MISTRAL_MODEL = "mistral-medium-latest"; - -async function configureProvidersInteractive(env: EnvMap): Promise<"recommended" | "custom"> { - p.note( - [ - "Elmo needs two kinds of providers:", - "", - pc.bold("1. A scraper") + " — to track ChatGPT and Google AI Mode (no public APIs):", - ` • ${pc.cyan("BrightData")} — cheap solid option, ~$0.45/mo per prompt`, - ` • ${pc.cyan("Olostep")} — premium option, powers Peec/AirOps, ~$2.25/mo per prompt`, - "", - pc.bold("2. A direct LLM API") + " — for low-latency tasks (onboarding analysis, sentiment scoring,", - " ad-hoc LLM calls). Required:", - ` • ${pc.cyan("OpenRouter")} — one key, all major models (recommended)`, - ` • ${pc.cyan("Anthropic / OpenAI / Mistral")} — direct provider keys`, - "", - "Pricing assumes Elmo's default cadence (5 runs/day × 2 surfaces).", - ].join("\n"), - "AI providers", - ); - - const mode = await p.select({ - message: "Setup mode", - options: [ - { value: "recommended" as const, label: "Recommended — one scraper + one direct API" }, - { value: "custom" as const, label: "Custom — pick each provider individually" }, - ], - initialValue: "recommended" as const, - }); - assertNotCancelled(mode); - - if (mode === "recommended") { - await configureProvidersRecommended(env); - } else { - await configureProvidersCustom(env); - } - return mode; -} - -async function configureProvidersRecommended(env: EnvMap): Promise { - const targets: string[] = []; - - // ── Scraper ───────────────────────────────────────────────────────────── - const scraper = await p.select({ - message: "Scraper (tracks ChatGPT + Google AI Mode)", - options: [ - { value: "brightdata" as const, label: "BrightData — ~$0.45/mo per prompt (cheaper)" }, - { value: "olostep" as const, label: "Olostep — ~$2.25/mo per prompt (premium)" }, - ], - initialValue: "brightdata" as const, - }); - assertNotCancelled(scraper); - await collectScraperKey(scraper, env); - for (const model of DEFAULT_SCRAPER_MODELS) { - targets.push(formatScrapeTarget({ model, provider: scraper, webSearch: true })); - } - - // ── Direct API ────────────────────────────────────────────────────────── - const direct = await p.select({ - message: "Direct LLM API (powers onboarding analysis + sentiment scoring)", - options: [ - { value: "openrouter" as const, label: "OpenRouter — one key, all major models (recommended)" }, - { value: "anthropic" as const, label: "Anthropic — direct Claude" }, - { value: "openai" as const, label: "OpenAI — direct GPT-* models" }, - { value: "mistral" as const, label: "Mistral — direct Mistral models" }, - ], - initialValue: "openrouter" as const, - }); - assertNotCancelled(direct); - await collectDirectApiQuick(direct, env); - - await finalizeScrapeTargets(env, targets, { skipEdit: true }); -} - -async function configureProvidersCustom(env: EnvMap): Promise { - const targets: string[] = []; - - p.log.step(pc.bold("Step 1 of 2 — Direct LLM API (at least one is required)")); - // Order matches the auto-pick preference in onboarding/llm.ts so the first - // provider asked is the one onboarding will reach for by default. - while (!hasDirectApiConfigured(env)) { - await collectOpenRouter(env, targets); - await collectAnthropic(env, targets); - await collectOpenAI(env, targets); - await collectMistral(env, targets); - if (!hasDirectApiConfigured(env)) { - p.log.warn( - "Onboarding analysis and other low-latency LLM tasks require a direct API. Configure at least one before continuing.", - ); - } - } - - p.log.step(pc.bold("Step 2 of 2 — Scrapers (optional, but needed to track ChatGPT / Google AI Mode)")); - await collectBrightData(env, targets); - await collectOlostep(env, targets); - await collectDataForSEO(env, targets); - - await finalizeScrapeTargets(env, targets); -} - -function hasDirectApiConfigured(env: EnvMap): boolean { - return Boolean(env.ANTHROPIC_API_KEY || env.OPENAI_API_KEY || env.MISTRAL_API_KEY || env.OPENROUTER_API_KEY); -} - -async function collectScraperKey(scraper: "brightdata" | "olostep", env: EnvMap): Promise { - if (scraper === "brightdata") { - p.log.info(`Sign up: ${link(pc.cyan(BRIGHTDATA_AFFILIATE), BRIGHTDATA_AFFILIATE)}`); - const key = await p.password({ - message: "BrightData API token", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.BRIGHTDATA_API_TOKEN = key; - } else { - p.log.info(`Sign up: ${link(pc.cyan(OLOSTEP_AFFILIATE), OLOSTEP_AFFILIATE)}`); - const key = await p.password({ - message: "Olostep API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.OLOSTEP_API_KEY = key; - } -} - -async function collectDirectApiQuick( - kind: "openrouter" | "anthropic" | "openai" | "mistral", - env: EnvMap, -): Promise { - if (kind === "openrouter") { - const key = await p.password({ - message: "OpenRouter API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.OPENROUTER_API_KEY = key; - } else if (kind === "anthropic") { - const key = await p.password({ - message: "Anthropic API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.ANTHROPIC_API_KEY = key; - } else if (kind === "openai") { - const key = await p.password({ - message: "OpenAI API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.OPENAI_API_KEY = key; - } else { - const key = await p.password({ - message: "Mistral API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.MISTRAL_API_KEY = key; - } -} - -async function collectBrightData(env: EnvMap, targets: string[]): Promise { - const enable = await p.confirm({ - message: `Configure ${pc.bold("BrightData")}? (~$0.45/mo per prompt)`, - initialValue: true, - }); - assertNotCancelled(enable); - if (!enable) return; - - p.log.info(`Sign up and generate an API token: ${link(pc.cyan(BRIGHTDATA_AFFILIATE), BRIGHTDATA_AFFILIATE)}`); - const key = await p.password({ - message: "BrightData API token", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.BRIGHTDATA_API_TOKEN = key; - - await pickScraperTargets({ - providerLabel: "BrightData", - providerId: "brightdata", - allModels: BRIGHTDATA_MODELS as readonly string[], - targets, - }); -} - -async function collectOlostep(env: EnvMap, targets: string[]): Promise { - const enable = await p.confirm({ - message: `Configure ${pc.bold("Olostep")}? (~$2.25/mo per prompt)`, - initialValue: false, - }); - assertNotCancelled(enable); - if (!enable) return; - - p.log.info(`Grab an API key: ${link(pc.cyan(OLOSTEP_AFFILIATE), OLOSTEP_AFFILIATE)}`); - const key = await p.password({ - message: "Olostep API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.OLOSTEP_API_KEY = key; - - await pickScraperTargets({ - providerLabel: "Olostep", - providerId: "olostep", - allModels: OLOSTEP_MODELS as readonly string[], - targets, - }); -} - -async function pickScraperTargets(args: { - providerLabel: string; - providerId: "brightdata" | "olostep"; - allModels: readonly string[]; - targets: string[]; -}): Promise { - const selected = (await p.multiselect({ - message: `LLM Providers to track via ${args.providerLabel}`, - options: args.allModels.map((model) => ({ value: model, label: model })), - required: true, - initialValues: [...DEFAULT_SCRAPER_MODELS], - })) as string[] | symbol; - assertNotCancelled(selected); - - for (const model of selected) { - args.targets.push(formatScrapeTarget({ model, provider: args.providerId, webSearch: true })); - } -} - -async function collectAnthropic(env: EnvMap, targets: string[]): Promise { - const enable = await p.confirm({ - message: `Configure ${pc.bold("Anthropic API")}? (direct Claude — ~$4–5/mo per prompt per model)`, - initialValue: false, - }); - assertNotCancelled(enable); - if (!enable) return; - - const key = await p.password({ - message: "Anthropic API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.ANTHROPIC_API_KEY = key; - - const model = await p.text({ - message: "Claude model", - placeholder: DEFAULT_ANTHROPIC_MODEL, - defaultValue: DEFAULT_ANTHROPIC_MODEL, - }); - assertNotCancelled(model); - const slug = model || DEFAULT_ANTHROPIC_MODEL; - - const webSearch = await p.confirm({ - message: "Enable web search? (recommended, but more expensive)", - initialValue: true, - }); - assertNotCancelled(webSearch); - - targets.push(formatScrapeTarget({ model: "claude", provider: "anthropic-api", version: slug, webSearch })); -} - -async function collectOpenAI(env: EnvMap, targets: string[]): Promise { - const enable = await p.confirm({ - message: `Configure ${pc.bold("OpenAI API")}? (gpt-* with web search — not the real ChatGPT UI)`, - initialValue: false, - }); - assertNotCancelled(enable); - if (!enable) return; - - const key = await p.password({ - message: "OpenAI API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.OPENAI_API_KEY = key; - - const model = await p.text({ - message: "OpenAI model", - placeholder: DEFAULT_OPENAI_MODEL, - defaultValue: DEFAULT_OPENAI_MODEL, - }); - assertNotCancelled(model); - const slug = model || DEFAULT_OPENAI_MODEL; - - const webSearch = await p.confirm({ - message: "Enable web search? (recommended, but more expensive)", - initialValue: true, - }); - assertNotCancelled(webSearch); - - targets.push(formatScrapeTarget({ model: "chatgpt", provider: "openai-api", version: slug, webSearch })); -} - -async function collectMistral(env: EnvMap, targets: string[]): Promise { - const enable = await p.confirm({ - message: `Configure ${pc.bold("Mistral API")}? (direct Mistral models)`, - initialValue: false, - }); - assertNotCancelled(enable); - if (!enable) return; - - const key = await p.password({ - message: "Mistral API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.MISTRAL_API_KEY = key; - - const model = await p.text({ - message: "Mistral model", - placeholder: DEFAULT_MISTRAL_MODEL, - defaultValue: DEFAULT_MISTRAL_MODEL, - }); - assertNotCancelled(model); - const slug = model || DEFAULT_MISTRAL_MODEL; - - const webSearch = await p.confirm({ - message: "Enable web search? (recommended, but more expensive)", - initialValue: true, - }); - assertNotCancelled(webSearch); - - targets.push(formatScrapeTarget({ model: "mistral", provider: "mistral-api", version: slug, webSearch })); -} - -async function collectOpenRouter(env: EnvMap, targets: string[]): Promise { - const enable = await p.confirm({ - message: `Configure ${pc.bold("OpenRouter")}? (one key, many hosted models)`, - initialValue: false, - }); - assertNotCancelled(enable); - if (!enable) return; - - const key = await p.password({ - message: "OpenRouter API key", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(key); - env.OPENROUTER_API_KEY = key; - - const model = await p.text({ - message: "OpenRouter model slug", - placeholder: DEFAULT_OPENROUTER_MODEL, - defaultValue: DEFAULT_OPENROUTER_MODEL, - }); - assertNotCancelled(model); - const slug = model || DEFAULT_OPENROUTER_MODEL; - - const webSearch = await p.confirm({ - message: "Enable web search? (recommended, but more expensive)", - initialValue: true, - }); - assertNotCancelled(webSearch); - - targets.push(formatScrapeTarget({ model: "claude", provider: "openrouter", version: slug, webSearch })); -} - -async function collectDataForSEO(env: EnvMap, targets: string[]): Promise { - const enable = await p.confirm({ - message: `Configure ${pc.bold("DataForSEO")}? (Google AI Mode scraping)`, - initialValue: false, - }); - assertNotCancelled(enable); - if (!enable) return; - - const login = await p.text({ - message: "DataForSEO login", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(login); - env.DATAFORSEO_LOGIN = login; - - const pwd = await p.password({ - message: "DataForSEO password", - validate: (v) => (!v ? "Required" : undefined), - }); - assertNotCancelled(pwd); - env.DATAFORSEO_PASSWORD = pwd; - - const addTarget = await p.confirm({ - message: "Also scrape Google AI Mode via DataForSEO? (google-ai-mode:dataforseo:online)", - initialValue: false, - }); - assertNotCancelled(addTarget); - if (addTarget) { - targets.push(formatScrapeTarget({ model: "google-ai-mode", provider: "dataforseo", webSearch: true })); - } -} - -async function finalizeScrapeTargets( - env: EnvMap, - targets: string[], - options: { skipEdit?: boolean } = {}, -): Promise { - const deduped = dedupeTargets(targets); - - if (!deduped) { - p.log.warn("No SCRAPE_TARGETS configured. Elmo will not run scheduled checks until you set them."); - p.log.info(`Reference: ${link(pc.cyan(PROVIDERS_DOC_URL), PROVIDERS_DOC_URL)}`); - - const addManual = await p.confirm({ - message: "Enter SCRAPE_TARGETS manually now?", - initialValue: false, - }); - assertNotCancelled(addManual); - if (addManual) { - const manual = await p.text({ - message: "SCRAPE_TARGETS (model:provider[:version][:online], comma-separated)", - placeholder: "chatgpt:brightdata:online,google-ai-mode:brightdata:online", - validate: validateScrapeTargetsInput, - }); - assertNotCancelled(manual); - env.SCRAPE_TARGETS = manual; - } - return; - } - - if (options.skipEdit) { - env.SCRAPE_TARGETS = deduped; - return; - } - - const customize = await p.confirm({ - message: "Edit SCRAPE_TARGETS before saving?", - initialValue: false, - }); - assertNotCancelled(customize); - - if (customize) { - p.log.info(`Reference: ${link(pc.cyan(PROVIDERS_DOC_URL), PROVIDERS_DOC_URL)}`); - const manual = await p.text({ - message: "SCRAPE_TARGETS", - initialValue: deduped, - validate: validateScrapeTargetsInput, - }); - assertNotCancelled(manual); - env.SCRAPE_TARGETS = manual; - p.log.step(`SCRAPE_TARGETS:\n ${pc.cyan(manual)}`); - } else { - env.SCRAPE_TARGETS = deduped; - } -} - -function validateScrapeTargetsInput(value: string | undefined): string | undefined { - if (!value) return "Required"; - try { - parseScrapeTargets(value); - } catch (error) { - return error instanceof Error ? error.message.split("\n")[0] : String(error); - } - return undefined; -} - -function dedupeTargets(targets: string[]): string { - const seen = new Set(); - const out: string[] = []; - for (const t of targets) { - if (seen.has(t)) continue; - seen.add(t); - out.push(t); - } - return out.join(","); -} - -// ── Start helper (used by init) ────────────────────────────────────────────── - -async function doStart(configDir: string): Promise { - assertDockerRunning(); - - log.step("Starting Docker Compose stack..."); - await runDockerCompose(configDir, ["up", "-d"]); - - const s = p.spinner(); - s.start("Waiting for services to become healthy..."); - const ok = await waitForHealthy(configDir, 180_000); - if (ok) { - s.stop("All services healthy!"); - } else { - s.stop("Health check timed out."); - p.log.warn("Some services did not report healthy status."); - } - - log.info("Examples:"); - console.log(` ${pc.bold("elmo compose logs -f")}`); - console.log(` ${pc.bold("elmo compose logs -f web")}`); - console.log(` ${pc.bold("elmo compose ps")}`); - console.log(` ${pc.bold("elmo compose down")}`); -} - -// ── Command: compose ───────────────────────────────────────────────────────── - -async function runCompose(args: string[], options: DirOption): Promise { - const configDir = await resolveConfigDir(options.dir); - assertDockerRunning(); - await runDockerCompose(configDir, args); -} - -// ── Command: edit ──────────────────────────────────────────────────────────── - -async function runEdit(target: string, options: DirOption): Promise { - const configDir = await resolveConfigDir(options.dir); - - let filePath: string; - if (target === "env") { - filePath = path.join(configDir, ".env"); - } else if (target === "compose") { - filePath = path.join(configDir, "elmo.yaml"); - } else { - throw new Error(`Unknown edit target: ${target}. Use \`env\` or \`compose\`.`); - } - - if (!(await fileExists(filePath))) { - throw new Error(`File not found: ${filePath}`); - } - - const editorEnv = process.env.VISUAL || process.env.EDITOR || "nano"; - const parts = editorEnv.split(/\s+/).filter(Boolean); - const cmd = parts[0] ?? "nano"; - const args = [...parts.slice(1), filePath]; - - await new Promise((resolve, reject) => { - const child = spawn(cmd, args, { stdio: "inherit" }); - child.on("close", (code) => { - if (code === 0) resolve(); - else reject(new Error(`${cmd} exited with code ${code}`)); - }); - child.on("error", (err) => reject(err)); - }); - - log.info("Restart the stack with `elmo compose up -d` to apply changes."); -} - -// ── Compose YAML Builder ───────────────────────────────────────────────────── - -function buildComposeYaml(options: { - dev: boolean; - postgresMode: PostgresMode; - repoRoot: string; - dockerDir?: string; - port: number; - version: string; -}): string { - const services: string[] = []; - const volumes = new Set(); - - const dependsOnWeb: string[] = []; - const dependsOnWorker: string[] = []; - - const dependencyConditions: Record = { - postgres: "service_healthy", - "db-migrate": "service_completed_successfully", - }; - - const dockerfilePath = options.dockerDir - ? path.relative(options.repoRoot, path.join(options.dockerDir, "Dockerfile")) - : "docker/Dockerfile"; - - if (options.postgresMode === "docker") { - services.push(buildPostgresService()); - services.push( - buildDbMigrateService({ - dev: options.dev, - dockerfilePath, - repoRoot: options.repoRoot, - version: options.version, - }), - ); - dependsOnWeb.push("db-migrate"); - dependsOnWorker.push("db-migrate"); - volumes.add("postgres_data"); - } - - services.push( - buildWebService({ - dev: options.dev, - dependsOn: dependsOnWeb, - dependencyConditions, - repoRoot: options.repoRoot, - dockerfilePath, - port: options.port, - version: options.version, - }), - ); - services.push( - buildWorkerService({ - dev: options.dev, - dependsOn: dependsOnWorker, - dependencyConditions, - repoRoot: options.repoRoot, - dockerfilePath, - version: options.version, - }), - ); - - const lines = [renderedByHeader(options.version), "", "name: elmo", "", "services:"]; - lines.push(...services.map((service) => indentBlock(service, 2))); - - if (volumes.size > 0) { - lines.push("", "volumes:"); - for (const volume of volumes) { - lines.push(` ${volume}:`); - } - } - - return `${lines.join("\n")}\n`; -} - -function buildPostgresService(): string { - return [ - "postgres:", - " image: postgres:16-alpine", - " environment:", - " POSTGRES_USER: postgres", - " POSTGRES_PASSWORD: postgres", - " POSTGRES_DB: elmo", - " volumes:", - " - postgres_data:/var/lib/postgresql/data", - " ports:", - ' - "5432:5432"', - " healthcheck:", - ' test: ["CMD-SHELL", "pg_isready -U postgres"]', - " interval: 5s", - " timeout: 5s", - " retries: 5", - " start_period: 30s", - ].join("\n"); -} - -function buildDbMigrateService(options: { - dev: boolean; - dockerfilePath: string; - repoRoot: string; - version: string; -}): string { - const lines = ["db-migrate:"]; - if (options.dev) { - lines.push( - " build:", - ` context: ${options.repoRoot}`, - ` dockerfile: ${options.dockerfilePath}`, - " target: migrate", - ); - } else { - lines.push(` image: elmohq/elmo-db-migrate:${options.version}`); - } - - lines.push( - " environment:", - " - DATABASE_URL=postgres://postgres:postgres@postgres:5432/elmo", - " depends_on:", - " postgres:", - " condition: service_healthy", - ); - - return lines.join("\n"); -} - -function buildWebService(options: { - dev: boolean; - dependsOn: string[]; - dependencyConditions: Record; - repoRoot: string; - dockerfilePath: string; - port: number; - version: string; -}): string { - const lines = ["web:"]; - if (options.dev) { - lines.push( - " build:", - ` context: ${options.repoRoot}`, - ` dockerfile: ${options.dockerfilePath}`, - " target: web", - " args:", - " DEPLOYMENT_MODE: local", - ); - } else { - lines.push(` image: elmohq/elmo-web:${options.version}`); - } - - lines.push(" env_file:", " - path: .env", " required: true", " ports:", ` - "${options.port}:3000"`); - - if (options.dependsOn.length > 0) { - lines.push(" depends_on:"); - for (const service of options.dependsOn) { - const condition = options.dependencyConditions[service] ?? "service_started"; - lines.push(` ${service}:`, ` condition: ${condition}`); - } - } - - return lines.join("\n"); -} - -function buildWorkerService(options: { - dev: boolean; - dependsOn: string[]; - dependencyConditions: Record; - repoRoot: string; - dockerfilePath: string; - version: string; -}): string { - const lines = ["worker:"]; - if (options.dev) { - lines.push( - " build:", - ` context: ${options.repoRoot}`, - ` dockerfile: ${options.dockerfilePath}`, - " target: worker", - " args:", - " DEPLOYMENT_MODE: local", - ); - } else { - lines.push(` image: elmohq/elmo-worker:${options.version}`); - } - - lines.push(" env_file:", " - path: .env", " required: true"); - - if (options.dependsOn.length > 0) { - lines.push(" depends_on:"); - for (const service of options.dependsOn) { - const condition = options.dependencyConditions[service] ?? "service_started"; - lines.push(` ${service}:`, ` condition: ${condition}`); - } - } - - return lines.join("\n"); -} - -function indentBlock(block: string, spaces: number): string { - const indent = " ".repeat(spaces); - return block - .split("\n") - .map((line) => `${indent}${line}`) - .join("\n"); -} - -// ── Docker Helpers ─────────────────────────────────────────────────────────── - -async function getComposeServices(configDir: string): Promise { - const output = await runDockerComposeCapture(configDir, ["ps", "--format", "json"]); - if (!output.trim()) { - return []; - } - try { - const trimmed = output.trim(); - const parsed = JSON.parse(trimmed); - if (Array.isArray(parsed)) { - return parsed as ComposeService[]; - } - if (typeof parsed === "object" && parsed !== null) { - return [parsed as ComposeService]; - } - return []; - } catch { - try { - return output - .trim() - .split("\n") - .filter((line) => line.trim()) - .map((line) => JSON.parse(line) as ComposeService); - } catch { - log.warn("Unable to parse docker compose status."); - return []; - } - } -} - -function isServiceReady(service: ComposeService): boolean { - if (service.Health) { - return service.Health === "healthy"; - } - if (service.State?.startsWith("running")) { - return true; - } - return false; -} - -async function waitForHealthy(configDir: string, timeoutMs: number): Promise { - const start = Date.now(); - while (Date.now() - start < timeoutMs) { - const services = await getComposeServices(configDir); - if (services.length > 0 && services.every(isServiceReady)) { - return true; - } - await sleep(3000); - } - return false; -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function runDockerCompose(configDir: string, args: string[]): Promise { - return new Promise((resolve, reject) => { - const composeFile = path.join(configDir, "elmo.yaml"); - const commandArgs = ["compose", "-f", composeFile, ...args]; - const child = spawn("docker", commandArgs, { - stdio: "inherit", - }); - child.on("close", (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`docker compose exited with code ${code}`)); - } - }); - }); -} - -function runDockerComposeCapture(configDir: string, args: string[]): Promise { - return new Promise((resolve, reject) => { - const composeFile = path.join(configDir, "elmo.yaml"); - const commandArgs = ["compose", "-f", composeFile, ...args]; - const child = spawn("docker", commandArgs); - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (data: Buffer) => { - stdout += data.toString(); - }); - child.stderr.on("data", (data: Buffer) => { - stderr += data.toString(); - }); - child.on("close", (code) => { - if (code === 0) { - resolve(stdout); - } else { - reject(new Error(stderr || `docker compose exited with code ${code}`)); - } - }); - }); -} - -function assertDockerRunning(): void { - const result = spawnSync("docker", ["info"], { - stdio: "ignore", - }); - if (result.status !== 0) { - throw new Error("Docker does not appear to be running. Start Docker and try again."); - } -} - -// ── Docker Dir Resolution ──────────────────────────────────────────────────── - -async function resolveDockerDirInteractive(cwd: string): Promise { - const inCwd = await fileExists(path.join(cwd, "Dockerfile")); - const inDockerDir = await fileExists(path.join(cwd, "docker", "Dockerfile")); - const defaultDir = inCwd ? "." : inDockerDir ? "docker" : "."; - - const dir = await p.text({ - message: "Path to docker directory (contains Dockerfile)", - defaultValue: defaultDir, - }); - assertNotCancelled(dir); - - const resolved = path.resolve(cwd, dir); - if (!(await fileExists(path.join(resolved, "Dockerfile")))) { - p.log.error(`Dockerfile not found in ${resolved}. Provide the directory that contains Dockerfile.`); - process.exit(1); - } - - return resolved; -} - -async function resolveDockerDirAuto(cwd: string, explicitDir?: string): Promise { - if (explicitDir) { - const resolved = path.resolve(cwd, explicitDir); - if (!(await fileExists(path.join(resolved, "Dockerfile")))) { - throw new Error(`Dockerfile not found in ${resolved}`); - } - return resolved; - } - - // Auto-detect - if (await fileExists(path.join(cwd, "docker", "Dockerfile"))) { - return path.resolve(cwd, "docker"); - } - if (await fileExists(path.join(cwd, "Dockerfile"))) { - return cwd; - } - - throw new Error("Could not find Dockerfile. Specify --docker-dir or set ELMO_DOCKER_DIR."); -} - -// ── Config Dir Resolution ──────────────────────────────────────────────────── - -async function resolveConfigDir(explicitDir?: string): Promise { - const resolved = explicitDir ? path.resolve(process.cwd(), explicitDir) : CONFIG_HOME; - const composePath = path.join(resolved, "elmo.yaml"); - if (!(await fileExists(composePath))) { - if (explicitDir) { - throw new Error( - `Config directory does not contain elmo.yaml: ${resolved}\nRun \`elmo init --dir ${explicitDir}\` to create it.`, - ); - } - throw new Error(`No config found at ${resolved}. Run \`elmo init\` to create one, or specify --dir.`); - } - return resolved; -} - -// ── File & Config Helpers ──────────────────────────────────────────────────── - -async function writeConfigFiles( - configDir: string, - initConfig: { - env: EnvMap; - composeYaml: string; - postgresMode: PostgresMode; - dev: boolean; - version: string; - }, -): Promise { - const envPath = path.join(configDir, ".env"); - const composePath = path.join(configDir, "elmo.yaml"); - - await ensureDir(configDir); - await fs.writeFile(envPath, buildEnvFile(initConfig.env, initConfig.version), "utf8"); - await fs.writeFile(composePath, initConfig.composeYaml, "utf8"); -} - -function renderedByHeader(version: string): string { - return [ - `# Rendered by elmo ${version} on ${new Date().toISOString()}`, - "# Re-run `elmo init` after upgrading the CLI to refresh this file.", - ].join("\n"); -} - -function buildEnvFile(env: EnvMap, version: string): string { - const lines = [renderedByHeader(version), "# WARNING: contains secrets. Do not commit.", ""]; - - for (const [key, rawValue] of Object.entries(env)) { - if (rawValue === undefined) { - continue; - } - lines.push(`${key}=${formatEnvValue(rawValue)}`); - } - - return `${lines.join("\n")}\n`; -} - -function formatEnvValue(value: string): string { - if (value === "") { - return '""'; - } - if (/[\s#"']/u.test(value)) { - const escaped = value.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); - return `"${escaped}"`; - } - return value; -} - -async function fileExists(target: string): Promise { - try { - await fs.access(target); - return true; - } catch { - return false; - } -} - -async function ensureDir(dir: string): Promise { - await fs.mkdir(dir, { recursive: true }); -} - -// ── Version Helpers ────────────────────────────────────────────────────────── - -async function getPackageVersion(): Promise { - const selfDir = path.dirname(fileURLToPath(import.meta.url)); - const packagePath = path.resolve(selfDir, "..", "package.json"); - const contents = await fs.readFile(packagePath, "utf8"); - const json = JSON.parse(contents) as { version?: string }; - return json.version!; -} - -async function maybeNotifyNewVersion(currentVersion: string): Promise { - try { - const response = await fetch("https://registry.npmjs.org/@elmohq/cli/latest"); - if (!response.ok) { - return; - } - const data = (await response.json()) as { - version?: string; - }; - if (!data.version) { - return; - } - if (semver.valid(currentVersion) && semver.lt(currentVersion, data.version)) { - log.warn(`New CLI version available (${data.version}). Run: npm install -g @elmohq/cli@latest`); - } - } catch { - // Ignore update errors - } -} - -// ── Entry Point ────────────────────────────────────────────────────────────── - main().catch((error) => { const msg = error instanceof Error ? error.message : String(error); console.error(`\n${pc.red("Error:")} ${msg}`); diff --git a/apps/worker/src/jobs/process-prompt.ts b/apps/worker/src/jobs/process-prompt.ts index b46efd20..d9d03555 100644 --- a/apps/worker/src/jobs/process-prompt.ts +++ b/apps/worker/src/jobs/process-prompt.ts @@ -3,6 +3,7 @@ import { db } from "@workspace/lib/db/db"; import { brands, citations, competitors, promptRuns, prompts, type Brand, type Competitor } from "@workspace/lib/db/schema"; import { eq } from "drizzle-orm"; import { RUNS_PER_PROMPT, getDefaultDelayHours } from "@workspace/lib/constants"; +import { analyzeMentions } from "@workspace/lib/mentions"; import { getProvider, parseScrapeTargets, @@ -102,48 +103,6 @@ async function getPromptContext(promptId: string): Promise }; } -function extractDomainFromUrl(urlOrDomain: string): string { - try { - const url = new URL(urlOrDomain.startsWith("http") ? urlOrDomain : `https://${urlOrDomain}`); - return url.hostname.replace(/^www\./, "").toLowerCase(); - } catch { - return urlOrDomain.replace(/^www\./, "").toLowerCase(); - } -} - -function analyzeMentions( - content: string, - brand: Brand, - competitorsList: Competitor[], -): { - brandMentioned: boolean; - competitorsMentioned: string[]; -} { - const contentLower = content.toLowerCase(); - - const brandNames = [brand.name, ...(brand.aliases || [])].map((n) => n.toLowerCase()); - const brandDomains = [ - extractDomainFromUrl(brand.website), - ...(brand.additionalDomains || []).map(extractDomainFromUrl), - ]; - const brandMentioned = - brandNames.some((n) => contentLower.includes(n)) || - brandDomains.some((d) => contentLower.includes(d)); - - const competitorsMentioned = competitorsList - .filter((competitor) => { - const names = [competitor.name, ...(competitor.aliases || [])].map((n) => n.toLowerCase()); - const nameMatch = names.some((n) => contentLower.includes(n)); - const domainMatch = (competitor.domains || []).some((d) => - contentLower.includes(extractDomainFromUrl(d)), - ); - return nameMatch || domainMatch; - }) - .map((competitor) => competitor.name); - - return { brandMentioned, competitorsMentioned }; -} - async function savePromptRun( promptId: string, brandId: string, diff --git a/apps/worker/src/report-worker.ts b/apps/worker/src/report-worker.ts index 3afba9ee..e3e6fe96 100644 --- a/apps/worker/src/report-worker.ts +++ b/apps/worker/src/report-worker.ts @@ -2,6 +2,7 @@ import { db } from "@workspace/lib/db/db"; import { reports, type Brand, brands } from "@workspace/lib/db/schema"; import { eq } from "drizzle-orm"; import { RUNS_PER_PROMPT } from "@workspace/lib/constants"; +import { analyzeMentions } from "@workspace/lib/mentions"; import { getProvider, parseScrapeTargets, type ModelConfig } from "@workspace/lib/providers"; import { analyzeBrand } from "@workspace/lib/onboarding"; import { isPromptBranded, computeSystemTags } from "@workspace/lib/tag-utils"; @@ -170,8 +171,10 @@ function selectOptimalPrompts( return selectedPrompts; } -// Function to check for brand and competitor mentions -function analyzeMentions( +// Adapt the report's flat {name, domain} competitors to the shared +// mention-detection shape. Mention logic lives in @workspace/lib/mentions so +// day-to-day tracking, reports, and the CLI all agree. +function analyzeReportMentions( content: string, brandName: string, brandWebsite: string, @@ -180,31 +183,11 @@ function analyzeMentions( brandMentioned: boolean; competitorsMentioned: string[]; } { - const contentLower = content.toLowerCase(); - const brandNameLower = brandName.toLowerCase(); - - // Extract domain from brandWebsite using URL constructor - const url = new URL(brandWebsite.startsWith('http') ? brandWebsite : `https://${brandWebsite}`); - const domain = url.hostname.replace(/^www\./, '').toLowerCase(); - - // Check for brand mention (brand name or domain) - const brandMentioned = contentLower.includes(brandNameLower) || contentLower.includes(domain); - - // Check for competitor mentions (by name or domain) - const competitorsMentioned = competitors - .filter((competitor) => { - const nameMatch = contentLower.includes(competitor.name.toLowerCase()); - - // Extract domain from competitor website - const competitorUrl = new URL(competitor.domain.startsWith('http') ? competitor.domain : `https://${competitor.domain}`); - const competitorDomain = competitorUrl.hostname.replace(/^www\./, '').toLowerCase(); - - const domainMatch = contentLower.includes(competitorDomain); - return nameMatch || domainMatch; - }) - .map((competitor) => competitor.name); - - return { brandMentioned, competitorsMentioned }; + return analyzeMentions( + content, + { name: brandName, website: brandWebsite }, + competitors.map((c) => ({ name: c.name, domains: [c.domain] })), + ); } // Function to run a prompt across different models and return results. @@ -225,7 +208,7 @@ async function runPrompt( webSearch: config.webSearch, version: config.version, }); - const { brandMentioned, competitorsMentioned } = analyzeMentions( + const { brandMentioned, competitorsMentioned } = analyzeReportMentions( result.textContent, brandName, brandWebsite, diff --git a/knip.json b/knip.json index dcf14e68..76e1b4f6 100644 --- a/knip.json +++ b/knip.json @@ -23,6 +23,17 @@ "ignore": [".source/**", "source.generated.ts"], "ignoreDependencies": ["@takumi-rs/wasm", "shiki"] }, + "apps/cli": { + "ignoreDependencies": [ + "@ai-sdk/anthropic", + "@ai-sdk/openai", + "@anthropic-ai/sdk", + "@brightdata/sdk", + "ai", + "dataforseo-client", + "olostep" + ] + }, "apps/worker": { "entry": ["scripts/test-provider.ts"] }, diff --git a/packages/docs/content/docs/user-guide/cli-lab.mdx b/packages/docs/content/docs/user-guide/cli-lab.mdx new file mode 100644 index 00000000..a894f44f --- /dev/null +++ b/packages/docs/content/docs/user-guide/cli-lab.mdx @@ -0,0 +1,132 @@ +--- +title: CLI Lab +description: Run Elmo's core analysis as one-off commands — generate prompts, evaluate them, and plan AEO improvements — straight from your terminal. +--- + +The `elmo lab` commands give you most of Elmo's intelligence in a one-off, no-database capacity. They reuse the providers and API keys you already configured with [`elmo init`](/docs/getting-started) (read from `~/.elmo/.env`), so you can generate tracking prompts, evaluate them across models, and get AEO recommendations without standing up the full stack. + +The three commands are designed to be run in sequence: + +```bash +elmo lab brainstorm # 1. discover what to track (prompts + competitors) +elmo lab eval [prompts] # 2. run them → responses, citations, mentions, share-of-voice, fan-out +elmo lab plan # 3. turn your content + results into AEO recommendations +``` + + +These commands call real provider APIs and cost money per run, just like scheduled tracking. They never touch a database and never need `DATABASE_URL`. + + +## Prerequisites + +Run [`elmo init`](/docs/getting-started) once to configure providers and write `~/.elmo/.env`. The lab commands read that file for provider keys and your `SCRAPE_TARGETS`. If you keep config elsewhere, point any command at it with `--dir `, or just export the keys (`OPENAI_API_KEY`, `BRIGHTDATA_API_TOKEN`, `SCRAPE_TARGETS`, …) in your shell. + +## Selecting a model & provider + +Every command targets models with the same `model:provider[:version][:online]` nomenclature used by [`SCRAPE_TARGETS`](/docs/user-guide/providers): + +``` +chatgpt:brightdata:online +google-ai-mode:brightdata:online +claude:anthropic-api:claude-sonnet-4-6:online +``` + +- **`brainstorm`** and **`plan`** do structured research, which only **direct‑API** providers support (Anthropic, OpenAI, OpenRouter, Mistral). Pass a single `-m`; omit it to use your configured default. +- **`eval`** works with any provider. `-m` is **repeatable**, so you can compare several at once. With no `-m`, it falls back to every entry in your `SCRAPE_TARGETS`. + +```bash +elmo lab eval --brand-file brand.json \ + -m chatgpt:brightdata:online \ + -m claude:anthropic-api:claude-sonnet-4-6:online +``` + +## Output + +By default every command writes its artifacts to the **current directory** and prints a short summary to stdout. Point somewhere else with `-o `, or pass `--stdout` to skip files and emit only to stdout (handy for piping). + +Structured data is written as `--format csv` (default) or `--format jsonl`; model responses are written as Markdown; and the **brand pack** is always `brand.json`. + +### The brand pack + +`brand.json` is the artifact that ties the commands together. `brainstorm` and `plan` produce it (brand name, aliases, domains, competitors, prompts); `eval` consumes it with `--brand-file` to get the brand and competitor context it needs for mentions and share-of-voice. + +## 1. Brainstorm prompts + +Generate a set of AI tracking prompts (mostly unbranded category/persona queries) and the brand's direct competitors: + +```bash +elmo lab brainstorm nike.com --count 30 --competitors 10 -o ./nike +``` + +Writes `brand.json`, `prompts.csv`, and `competitors.csv` to `./nike`. + +| Option | Description | +|---|---| +| `-c, --count ` | Number of prompts (default 30) | +| `--competitors ` | Number of competitors (default 10) | +| `-m, --model ` | Research provider (direct API) | + +## 2. Evaluate prompts + +Run each prompt N times against each target and capture everything: + +```bash +elmo lab eval --brand-file ./nike/brand.json \ + -m chatgpt:brightdata:online \ + -m claude:anthropic-api:claude-sonnet-4-6:online \ + --runs 5 -o ./nike-eval +``` + +Prompts can come from `--brand-file`, positional arguments, repeated `--prompt`, a `--prompts-file` (one per line), or stdin (`-`). Brand/competitor context for mentions can come from the brand pack or from `--brand`, `--brand-domain`, `--alias`, and repeatable `--competitor name:domain` flags. + +The output directory contains: + +``` +nike-eval/ +├── responses/ +│ └── 001-best-running-shoes/ +│ ├── chatgpt__brightdata__run-1.md +│ └── claude__anthropic-api__run-1.md +├── citations.csv # url, domain, title, prompt, model, run +├── mentions.csv # brand + competitor mentions per run +├── share-of-voice.csv # brand vs. competitors +├── fan-out.csv # the queries each model fanned out to +├── summary.md # human-readable rollup + content gaps +├── run.json # metadata + aggregates +└── index.html # ← open this to browse everything +``` + +`index.html` is a single self-contained file (no server, no internet) — open it in a browser to read every rendered response with its mention badges, citations, and fan-out queries, filterable by prompt or model. + + +**Query fan-out** is a byproduct of evaluation, not a separate command: whenever a target has web search (`:online`), `eval` records the sub-queries the model issued and aggregates them in `fan-out.csv`, the stdout summary, and the report. + + +| Option | Description | +|---|---| +| `-m, --model ` | Target (repeatable; default `SCRAPE_TARGETS`) | +| `-n, --runs ` | Replications per prompt per target (default 5) | +| `--brand-file ` | Brand pack for prompts + mention context | +| `--competitor ` | Add a competitor (repeatable) | +| `--concurrency ` | Max concurrent provider calls (default 4) | + +## 3. Plan improvements + +Turn your own content (and, optionally, a prior eval) into a prioritized set of AEO recommendations: + +```bash +elmo lab plan ./content --website nike.com \ + --brand-file ./nike/brand.json \ + --eval-dir ./nike-eval -o ./nike-plan +``` + +`plan` reads the files/directories you pass (plus an optional `--website` excerpt and `--eval-dir` to ground the advice in real results), then writes `plan.md`, `suggestions.csv`, a refreshed `competitors.csv`, and an augmented `brand.json`. Because it also surfaces competitors, its `brand.json` can feed straight back into `eval`. + +## End-to-end + +```bash +# discover → evaluate → plan +elmo lab brainstorm nike.com -o ./nike +elmo lab eval --brand-file ./nike/brand.json -m chatgpt:brightdata:online -o ./nike-eval +elmo lab plan ./content --brand-file ./nike/brand.json --eval-dir ./nike-eval -o ./nike-plan +``` diff --git a/packages/docs/content/docs/user-guide/meta.json b/packages/docs/content/docs/user-guide/meta.json index 3ba839cf..b1a2b1a1 100644 --- a/packages/docs/content/docs/user-guide/meta.json +++ b/packages/docs/content/docs/user-guide/meta.json @@ -3,6 +3,7 @@ "pages": [ "index", "providers", + "cli-lab", "registration", "brand-setup", "prompt-wizard", diff --git a/packages/lib/package.json b/packages/lib/package.json index cadc7f3b..7dfac512 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -17,6 +17,7 @@ "./auth/client": "./src/auth/client.ts", "./auth/permissions": "./src/auth/permissions.ts", "./constants": "./src/constants.ts", + "./mentions": "./src/mentions.ts", "./text-extraction": "./src/text-extraction.ts", "./dataforseo": "./src/dataforseo.ts", "./onboarding": "./src/onboarding/index.ts", diff --git a/packages/lib/src/mentions.test.ts b/packages/lib/src/mentions.test.ts new file mode 100644 index 00000000..7eca5e97 --- /dev/null +++ b/packages/lib/src/mentions.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { analyzeMentions, extractDomainFromUrl } from "./mentions"; + +describe("extractDomainFromUrl", () => { + it("strips protocol and www, lowercases", () => { + expect(extractDomainFromUrl("https://www.Nike.com/shoes")).toBe("nike.com"); + expect(extractDomainFromUrl("nike.com")).toBe("nike.com"); + expect(extractDomainFromUrl("WWW.Example.CO.UK")).toBe("example.co.uk"); + }); +}); + +describe("analyzeMentions", () => { + const brand = { + name: "Nike", + website: "nike.com", + aliases: ["Nike Inc"], + additionalDomains: ["nike.co.uk"], + }; + const competitors = [ + { name: "Adidas", domains: ["adidas.com"], aliases: [] }, + { name: "New Balance", domains: ["newbalance.com"], aliases: ["NB"] }, + ]; + + it("detects the brand by name (case-insensitive)", () => { + const r = analyzeMentions("I really like NIKE running shoes.", brand, competitors); + expect(r.brandMentioned).toBe(true); + expect(r.competitorsMentioned).toEqual([]); + }); + + it("detects the brand by domain", () => { + const r = analyzeMentions("See nike.co.uk for details.", brand, competitors); + expect(r.brandMentioned).toBe(true); + }); + + it("detects competitors by name, domain, and alias", () => { + const r = analyzeMentions("Compare adidas.com and NB to others.", brand, competitors); + expect(r.brandMentioned).toBe(false); + expect(r.competitorsMentioned).toEqual(["Adidas", "New Balance"]); + }); + + it("returns no mentions when nobody appears", () => { + const r = analyzeMentions("A generic answer about footwear.", brand, competitors); + expect(r.brandMentioned).toBe(false); + expect(r.competitorsMentioned).toEqual([]); + }); + + it("tolerates missing optional fields", () => { + const r = analyzeMentions("puma is great", { name: "Puma" }, [{ name: "Reebok" }]); + expect(r.brandMentioned).toBe(true); + expect(r.competitorsMentioned).toEqual([]); + }); +}); diff --git a/packages/lib/src/mentions.ts b/packages/lib/src/mentions.ts new file mode 100644 index 00000000..be7bf412 --- /dev/null +++ b/packages/lib/src/mentions.ts @@ -0,0 +1,73 @@ +/** + * Brand & competitor mention detection. + * + * Case-insensitive substring matching of a brand (its name, aliases, website, + * and additional domains) and competitors (name, aliases, domains) against a + * model's response text. This is the single source of truth shared by the + * worker's day-to-day prompt processing, the report worker, and the CLI's + * `elmo lab eval` command — keep all callers on this function so mention/SoV + * numbers stay identical across surfaces. + * + * The inputs are intentionally minimal structural shapes (not DB row types) so + * non-DB callers like the CLI can use them without importing the schema. + */ + +export interface MentionBrand { + name: string; + /** Primary website/hostname (with or without protocol). Optional. */ + website?: string; + /** Other names users call the brand (abbreviations, parent company, etc.). */ + aliases?: string[] | null; + /** Other hostnames the brand owns. */ + additionalDomains?: string[] | null; +} + +export interface MentionCompetitor { + name: string; + /** Hostnames owned by the competitor (with or without protocol). */ + domains?: string[] | null; + aliases?: string[] | null; +} + +/** Normalize a URL or bare hostname to a lowercase host with `www.` stripped. */ +export function extractDomainFromUrl(urlOrDomain: string): string { + try { + const url = new URL(urlOrDomain.startsWith("http") ? urlOrDomain : `https://${urlOrDomain}`); + return url.hostname.replace(/^www\./, "").toLowerCase(); + } catch { + return urlOrDomain.replace(/^www\./, "").toLowerCase(); + } +} + +export function analyzeMentions( + content: string, + brand: MentionBrand, + competitorsList: MentionCompetitor[], +): { + brandMentioned: boolean; + competitorsMentioned: string[]; +} { + const contentLower = content.toLowerCase(); + + const brandNames = [brand.name, ...(brand.aliases ?? [])].map((n) => n.toLowerCase()); + const brandDomains = [ + ...(brand.website ? [extractDomainFromUrl(brand.website)] : []), + ...(brand.additionalDomains ?? []).map(extractDomainFromUrl), + ]; + const brandMentioned = + brandNames.some((n) => n && contentLower.includes(n)) || brandDomains.some((d) => d && contentLower.includes(d)); + + const competitorsMentioned = competitorsList + .filter((competitor) => { + const names = [competitor.name, ...(competitor.aliases ?? [])].map((n) => n.toLowerCase()); + const nameMatch = names.some((n) => n && contentLower.includes(n)); + const domainMatch = (competitor.domains ?? []).some((d) => { + const domain = extractDomainFromUrl(d); + return domain && contentLower.includes(domain); + }); + return nameMatch || domainMatch; + }) + .map((competitor) => competitor.name); + + return { brandMentioned, competitorsMentioned }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 237e8dea..69cbc76d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,15 +48,39 @@ importers: apps/cli: dependencies: + '@ai-sdk/anthropic': + specifier: ^3.0.81 + version: 3.0.81(zod@4.4.3) + '@ai-sdk/openai': + specifier: ^3.0.68 + version: 3.0.68(zod@4.4.3) + '@anthropic-ai/sdk': + specifier: ^0.102.0 + version: 0.102.0(zod@4.4.3) + '@brightdata/sdk': + specifier: ^1.1.0 + version: 1.1.0 '@clack/prompts': specifier: ^1.5.1 version: 1.5.1 + ai: + specifier: ^6.0.197 + version: 6.0.197(zod@4.4.3) commander: specifier: ^15.0.0 version: 15.0.0 + dataforseo-client: + specifier: ^2.0.25 + version: 2.0.25 dotenv: specifier: 17.4.2 version: 17.4.2 + marked: + specifier: ^14.1.4 + version: 14.1.4 + olostep: + specifier: ^1.1.0 + version: 1.1.0 picocolors: specifier: ^1.1.1 version: 1.1.1 @@ -66,6 +90,9 @@ importers: semver: specifier: ^7.8.2 version: 7.8.2 + zod: + specifier: ^4.4.3 + version: 4.4.3 devDependencies: '@types/node': specifier: ^25.9.2 @@ -76,12 +103,18 @@ importers: '@workspace/config': specifier: workspace:* version: link:../../packages/config + '@workspace/lib': + specifier: workspace:* + version: link:../../packages/lib rolldown: specifier: ^1.1.0 version: 1.1.0 typescript: specifier: ^6.0.3 version: 6.0.3 + vitest: + specifier: ^4.1.8 + version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@25.9.2)(@vitest/browser-playwright@4.1.8)(@vitest/coverage-v8@4.1.8)(vite@8.0.16(@types/node@25.9.2)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) apps/web: dependencies: @@ -6123,6 +6156,11 @@ packages: markdown-table@3.0.4: resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} + marked@14.1.4: + resolution: {integrity: sha512-vkVZ8ONmUdPnjCKc5uTRvmkRbx4EAi2OkTOXmfTDhZz3OFqMNBM1oTTWwTr4HY4uAEojhzPf+Fy8F1DWa3Sndg==} + engines: {node: '>= 18'} + hasBin: true + mdast-util-find-and-replace@3.0.2: resolution: {integrity: sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==} @@ -12731,6 +12769,8 @@ snapshots: markdown-table@3.0.4: {} + marked@14.1.4: {} + mdast-util-find-and-replace@3.0.2: dependencies: '@types/mdast': 4.0.4