[codex] ROY-24: should we use sqlite to cache data or config for scale up later?

README.md

@@ -76,6 +76,8 @@ Optional:
 - `PIV_MAX_POLL_CYCLES` (optional; stop polling after this many cycles)
 - `PIV_EXIT_WHEN_IDLE` (optional; default `1`, set `0` to keep polling when no issues are found)
 - `PIV_DRY_RUN=1` to avoid Linear/GitHub mutations
+- `PIV_STATE_STORE` (optional; `json` default, or `sqlite` for database-backed run-state storage)
+- `PIV_SQLITE_PATH` (required when `PIV_STATE_STORE=sqlite`; absolute/relative path to run-state sqlite db file)
 - `PIV_DEV_MODE=1` to stream Codex stdout/stderr logs during runs
 - `CODEX_SANDBOX` (optional; leave empty to disable sandbox, or set `read-only`, `workspace-write`, `danger-full-access`)
 - `CODEX_MODEL_PLAN` (optional; overrides planning model)
@@ -117,3 +119,4 @@ bun test
 - Run with authenticated `gh` (`gh auth status`).
 - Codex uses the default CLI home unless you explicitly set `CODEX_HOME`.
 - Linear integration uses the official `@linear/sdk` client.
+- Config resolution stays file/env based in `src/config.ts`; only run-state persistence can use SQLite.

src/config.ts

@@ -91,6 +91,10 @@ function buildEnvBase(cwd: string): ProjectRuntimeConfig {
 			sandbox,
 			codexHome,
 		},
+		stateStore: {
+			type: normalizeStateStoreType(env.PIV_STATE_STORE),
+			sqlitePath: normalizeOptionalValue(env.PIV_SQLITE_PATH),
+		},
 		skills: {
 			plan: path.join(cwd, "skills", "piv-plan", "SKILL.md"),
 			implement: path.join(cwd, "skills", "piv-implement", "SKILL.md"),
@@ -217,6 +221,11 @@ function mergeRuntime(
 			...(rootDefaults.codex ?? {}),
 			...(project.codex ?? {}),
 		},
+		stateStore: {
+			...base.stateStore,
+			...(rootDefaults.stateStore ?? {}),
+			...(project.stateStore ?? {}),
+		},
 		skills: {
 			...base.skills,
 			...(rootDefaults.skills ?? {}),
@@ -292,6 +301,21 @@ function normalizeSandboxValue(
 	);
 }
 
+function normalizeStateStoreType(
+	input: string | undefined,
+): ProjectRuntimeConfig["stateStore"]["type"] {
+	if (!input) {
+		return "json";
+	}
+	const value = input.trim().toLowerCase();
+	if (value === "json" || value === "sqlite") {
+		return value;
+	}
+	throw new Error(
+		`Invalid PIV_STATE_STORE value '${input}'. Use json or sqlite.`,
+	);
+}
+
 function validateProject(project: ResolvedProjectConfig): void {
 	if (!project.linear.apiKey) {
 		throw new Error(`LINEAR_API_KEY is required for project '${project.id}'`);
@@ -327,4 +351,9 @@ function validateProject(project: ResolvedProjectConfig): void {
 			`Polling max cycles must be a positive integer for project '${project.id}'`,
 		);
 	}
+	if (project.stateStore.type === "sqlite" && !project.stateStore.sqlitePath) {
+		throw new Error(
+			`PIV_SQLITE_PATH is required when state store is sqlite for project '${project.id}'`,
+		);
+	}
 }

src/index.ts

@@ -2,7 +2,7 @@
 import { parseArgs } from "./args";
 import { getProjectById, loadConfig } from "./config";
 import { logger, normalizeError, setupProcessErrorHandlers } from "./logger";
-import { loadRunState, normalizeIssueKey } from "./state";
+import { createRunStateStore, normalizeIssueKey } from "./state";
 import { runWorkflow } from "./workflow";
 
 async function main(): Promise<void> {
@@ -41,7 +41,8 @@ async function main(): Promise<void> {
 			throw new Error(`Project '${command.projectId}' not found`);
 		}
 		const key = normalizeIssueKey(command.issueKey);
-		const state = await loadRunState(project.workspacePath, project.id, key);
+		const store = createRunStateStore(project);
+		const state = await store.load(project.id, key);
 		if (!state) {
 			process.stdout.write(
 				`No run state found for ${key} in project ${project.id}\n`,

src/state.ts

@@ -1,10 +1,18 @@
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
 import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
 import path from "node:path";
-import type { RunState, WorkflowStage } from "./types";
+import type { ResolvedProjectConfig, RunState, WorkflowStage } from "./types";
 
 const LEGACY_STATE_DIR = path.join(".piv-loop", "runs");
 const STATE_ROOT_DIR = path.join(".piv-loop", "projects");
 
+export interface RunStateStore {
+	load(projectId: string, issueKey: string): Promise<RunState | null>;
+	save(state: RunState): Promise<void>;
+	list(projectId: string): Promise<RunState[]>;
+}
+
 export function normalizeIssueKey(input: string): string {
 	const match = input.trim().match(/[A-Z]+-\d+/);
 	if (!match) {
@@ -27,62 +35,153 @@ export function stateFilePath(
 	);
 }
 
-export async function loadRunState(
-	cwd: string,
-	projectId: string,
-	issueKey: string,
-): Promise<RunState | null> {
-	const file = stateFilePath(cwd, projectId, issueKey);
-	try {
-		const raw = await readFile(file, "utf8");
-		return JSON.parse(raw) as RunState;
-	} catch {
-		if (projectId !== "default") {
-			return null;
-		}
-		const legacy = path.join(
-			cwd,
-			LEGACY_STATE_DIR,
-			`${normalizeIssueKey(issueKey)}.json`,
-		);
+function legacyStateFilePath(cwd: string, issueKey: string): string {
+	return path.join(
+		cwd,
+		LEGACY_STATE_DIR,
+		`${normalizeIssueKey(issueKey)}.json`,
+	);
+}
+
+class JsonRunStateStore implements RunStateStore {
+	constructor(private readonly cwd: string) {}
+
+	async load(projectId: string, issueKey: string): Promise<RunState | null> {
+		const file = stateFilePath(this.cwd, projectId, issueKey);
 		try {
-			const raw = await readFile(legacy, "utf8");
+			const raw = await readFile(file, "utf8");
 			return JSON.parse(raw) as RunState;
 		} catch {
+			if (projectId !== "default") {
+				return null;
+			}
+			try {
+				const raw = await readFile(
+					legacyStateFilePath(this.cwd, issueKey),
+					"utf8",
+				);
+				return JSON.parse(raw) as RunState;
+			} catch {
+				return null;
+			}
+		}
+	}
+
+	async save(state: RunState): Promise<void> {
+		const file = stateFilePath(this.cwd, state.projectId, state.issue.key);
+		await mkdir(path.dirname(file), { recursive: true });
+		state.updatedAt = new Date().toISOString();
+		await writeFile(file, `${JSON.stringify(state, null, 2)}\n`, "utf8");
+	}
+
+	async list(projectId: string): Promise<RunState[]> {
+		const dir = path.join(this.cwd, STATE_ROOT_DIR, projectId, "runs");
+		try {
+			const files = await readdir(dir);
+			const runs: RunState[] = [];
+			for (const file of files) {
+				if (!file.endsWith(".json")) {
+					continue;
+				}
+				const raw = await readFile(path.join(dir, file), "utf8");
+				runs.push(JSON.parse(raw) as RunState);
+			}
+			return runs;
+		} catch {
+			return [];
+		}
+	}
+}
+
+class SqliteRunStateStore implements RunStateStore {
+	private readonly db: Database;
+
+	constructor(dbFile: string) {
+		mkdirSync(path.dirname(dbFile), { recursive: true });
+		this.db = new Database(dbFile, { create: true });
+		this.db.exec(
+			`CREATE TABLE IF NOT EXISTS run_states (
+				project_id TEXT NOT NULL,
+				issue_key TEXT NOT NULL,
+				state_json TEXT NOT NULL,
+				updated_at TEXT NOT NULL,
+				PRIMARY KEY (project_id, issue_key)
+			)`,
+		);
+		this.db.exec(
+			"CREATE INDEX IF NOT EXISTS idx_run_states_project_updated ON run_states(project_id, updated_at)",
+		);
+	}
+
+	async load(projectId: string, issueKey: string): Promise<RunState | null> {
+		const key = normalizeIssueKey(issueKey);
+		const row = this.db
+			.query(
+				"SELECT state_json FROM run_states WHERE project_id = ?1 AND issue_key = ?2",
+			)
+			.get(projectId, key) as { state_json: string } | null;
+		if (!row?.state_json) {
 			return null;
 		}
+		return JSON.parse(row.state_json) as RunState;
+	}
+
+	async save(state: RunState): Promise<void> {
+		state.updatedAt = new Date().toISOString();
+		const key = normalizeIssueKey(state.issue.key);
+		this.db
+			.query(
+				`INSERT INTO run_states (project_id, issue_key, state_json, updated_at)
+				 VALUES (?1, ?2, ?3, ?4)
+				 ON CONFLICT(project_id, issue_key) DO UPDATE SET
+				   state_json = excluded.state_json,
+				   updated_at = excluded.updated_at`,
+			)
+			.run(state.projectId, key, JSON.stringify(state), state.updatedAt);
+	}
+
+	async list(projectId: string): Promise<RunState[]> {
+		const rows = this.db
+			.query(
+				"SELECT state_json FROM run_states WHERE project_id = ?1 ORDER BY updated_at DESC",
+			)
+			.all(projectId) as Array<{ state_json: string }>;
+		return rows.map((row) => JSON.parse(row.state_json) as RunState);
 	}
 }
 
+export function createRunStateStore(
+	config: Pick<ResolvedProjectConfig, "workspacePath" | "stateStore">,
+): RunStateStore {
+	if (config.stateStore.type === "sqlite") {
+		const dbPath =
+			config.stateStore.sqlitePath ??
+			path.join(config.workspacePath, ".piv-loop", "run-state.sqlite");
+		return new SqliteRunStateStore(dbPath);
+	}
+	return new JsonRunStateStore(config.workspacePath);
+}
+
+export async function loadRunState(
+	cwd: string,
+	projectId: string,
+	issueKey: string,
+): Promise<RunState | null> {
+	return new JsonRunStateStore(cwd).load(projectId, issueKey);
+}
+
 export async function saveRunState(
 	cwd: string,
 	state: RunState,
 ): Promise<void> {
-	const file = stateFilePath(cwd, state.projectId, state.issue.key);
-	await mkdir(path.dirname(file), { recursive: true });
-	state.updatedAt = new Date().toISOString();
-	await writeFile(file, `${JSON.stringify(state, null, 2)}\n`, "utf8");
+	await new JsonRunStateStore(cwd).save(state);
 }
 
 export async function listRunStates(
 	cwd: string,
 	projectId: string,
 ): Promise<RunState[]> {
-	const dir = path.join(cwd, STATE_ROOT_DIR, projectId, "runs");
-	try {
-		const files = await readdir(dir);
-		const runs: RunState[] = [];
-		for (const file of files) {
-			if (!file.endsWith(".json")) {
-				continue;
-			}
-			const raw = await readFile(path.join(dir, file), "utf8");
-			runs.push(JSON.parse(raw) as RunState);
-		}
-		return runs;
-	} catch {
-		return [];
-	}
+	return new JsonRunStateStore(cwd).list(projectId);
 }
 
 export function transitionStage(

src/types.ts

@@ -78,6 +78,10 @@ export interface ProjectRuntimeConfig {
 		sandbox?: "read-only" | "workspace-write" | "danger-full-access";
 		codexHome?: string;
 	};
+	stateStore: {
+		type: "json" | "sqlite";
+		sqlitePath?: string;
+	};
 	skills: {
 		plan: string;
 		implement: string;