Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
"@dqbd/tiktoken": "^1.0.22",
"@electric-sql/pglite": "0.4.3",
"@jsquash/avif": "^2.1.1",
"@jsquash/png": "^3.1.1",
"@modelcontextprotocol/sdk": "1.29.0",
"ai": "^6.0.168",
"cookie-parser": "^1.4.7",
Expand All @@ -84,6 +85,7 @@
"express": "^5.1.0",
"express-rate-limit": "^7.5.0",
"gray-matter": "^4.0.3",
"heic-decode": "^2.1.0",
"marked": "^18.0.0",
"openai": "^4.0.0",
"pgvector": "^0.2.0",
Expand Down
8 changes: 6 additions & 2 deletions src/commands/migrations/v0_29_1.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ async function phaseBBackfill(opts: OrchestratorOpts): Promise<OrchestratorPhase
const { backfillEffectiveDate } = await import('../../core/backfill-effective-date.ts');
const cfg = loadConfig();
if (!cfg) throw new Error('No gbrain config; run `gbrain init` first.');
const engine = await createEngine(toEngineConfig(cfg));
const engineConfig = toEngineConfig(cfg);
const engine = await createEngine(engineConfig);
await engine.connect(engineConfig);

let totalExamined = 0;
let totalUpdated = 0;
Expand Down Expand Up @@ -82,7 +84,9 @@ async function phaseCVerify(opts: OrchestratorOpts): Promise<OrchestratorPhaseRe
const { loadConfig, toEngineConfig } = await import('../../core/config.ts');
const cfg = loadConfig();
if (!cfg) throw new Error('No gbrain config; run `gbrain init` first.');
const engine = await createEngine(toEngineConfig(cfg));
const engineConfig = toEngineConfig(cfg);
const engine = await createEngine(engineConfig);
await engine.connect(engineConfig);
// Count rows where effective_date is still NULL but frontmatter HAS a
// parseable date — those are the rows the backfill should have touched
// but didn't. (Rows that fall through to 'fallback' have non-null
Expand Down
25 changes: 9 additions & 16 deletions src/core/backfill-effective-date.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,14 +175,14 @@ export async function backfillEffectiveDate(
if (!opts.dryRun) {
// Compute effective_date for each row, then UPDATE in a batch wrapped
// in its own transaction (so SET LOCAL statement_timeout scopes to it).
// postgres.js's `transaction` would be cleaner but we're using executeRaw
// for engine portability; explicit BEGIN/COMMIT does the same on both.
if (isPostgres) {
await engine.executeRaw(`BEGIN`);
await engine.executeRaw(`SET LOCAL statement_timeout = '600s'`);
}
// postgres.js refuses bare BEGIN/COMMIT on pooled connections
// (UNSAFE_TRANSACTION); engine.transaction() routes through sql.begin()
// which uses a reserved backend.
await engine.transaction(async (tx) => {
if (isPostgres) {
await tx.executeRaw(`SET LOCAL statement_timeout = '600s'`);
}

try {
for (const r of rows) {
const fm = parseFrontmatter(r.frontmatter);
const filename = r.import_filename
Expand All @@ -204,21 +204,14 @@ export async function backfillEffectiveDate(

if (!opts.force && datesMatch && sourcesMatch) continue;

await engine.executeRaw(
await tx.executeRaw(
`UPDATE pages SET effective_date = $1::timestamptz, effective_date_source = $2 WHERE id = $3`,
[computed.date ? computed.date.toISOString() : null, computed.source, r.id],
);
touched++;
if (computed.source === 'fallback') fallback++;
}

if (isPostgres) await engine.executeRaw(`COMMIT`);
} catch (e) {
if (isPostgres) {
try { await engine.executeRaw(`ROLLBACK`); } catch { /* ignore */ }
}
throw e;
}
});
} else {
// Dry run: still count what WOULD change.
for (const r of rows) {
Expand Down