-
Notifications
You must be signed in to change notification settings - Fork 3
Expand file tree
/
Copy pathstore-postgres.ts
More file actions
2966 lines (2775 loc) · 157 KB
/
Copy pathstore-postgres.ts
File metadata and controls
2966 lines (2775 loc) · 157 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* PostgresStore — Production Store backed by PostgreSQL + pgvector
*
* ARCHITECTURE:
* • All projects share ONE PostgreSQL database.
* • Multi-tenancy: every table includes a `project_hash` column (first 16 hex chars
* of SHA-256(projectPath)). All queries filter by project_hash.
* • Vector search: pgvector `vector(768)` column + IVFFlat cosine index.
* • Full-text search: PostgreSQL tsvector/GIN index (replaces SQLite FTS5).
* • Hybrid search: BM25 (ts_rank) + cosine similarity, same formula as SqliteStore.
* • Hash chain: same SHA-256 chain logic, enforced by serialized INSERT via
* advisory locks (prevents concurrent writers racing on prev_hash).
* • RBAC: same token format (zcst.payload.hmac), same HMAC verification.
* Signing key stored in project_meta per project_hash.
*
* SECURITY:
* • All queries parameterized — no SQL injection possible.
* • project_hash is always derived server-side from projectPath — callers
* cannot supply an arbitrary hash to access another project's data.
* • Advisory lock (pg_advisory_xact_lock) serializes broadcast INSERTs per
* project to ensure hash chain integrity under concurrent writes.
* • Scrypt KDF for channel key (same as SqliteStore, same parameters).
* • Token HMAC verification is timing-safe (timingSafeEqual).
* • Row-level isolation: all queries include WHERE project_hash = $n.
*
* PERFORMANCE:
* • pg.Pool with configurable pool size (default 10 connections).
* • IVFFlat index on embeddings for O(√n) approximate cosine search.
* • GIN index on tsvector for O(log n) full-text search.
* • Complexity profile cached in project_meta (10-minute TTL, same as SqliteStore).
*/
import pg from "pg";
import { hashChannelKeyScrypt, verifyScryptHash, SCRYPT_PREFIX } from "./security/scrypt.js";
import { createHash, createHmac, randomBytes, randomUUID, scryptSync, timingSafeEqual } from "node:crypto";
import { Config } from "./config.js";
import { computeRowHash } from "./chain.js";
import { scheduleEventExtraction, supersedeEventEntries } from "./event_extractor.js";
import { getEmbedding, getEmbeddingQueued, cosineSimilarity, ACTIVE_MODEL } from "./embedder.js";
import { classifyFactKind, clampBroadcastSummary, clampWithMarker, MEMORY_KINDS, type EpistemicOpts } from "./memory.js";
import { isPinnedKind } from "./memory_quality.js";
import { verifyWrite, type VerifyResult } from "./effect_verify.js";
import { computeSalience, salienceEnabled } from "./salience.js";
import { budgetFacts, effectiveImportance } from "./recall_budget.js";
import { extractCoReferences, extractCoReferencesAsync, classifyRelation, graphMaxNodes } from "./indexing/community.js";
import { SIM_HIGH, MAX_SCAN_FACTS, detectConflict, autoResolveVictim } from "./contradiction_heuristics.js";
import { llmExtractEntities, entityEdgesFor, ENTITY_EXTRACT_ENABLED, ENTITY_BUDGET } from "./indexing/entity_extract.js";
import { detectCommunitiesFromRows } from "./indexing/community.js";
import { summarizeCommunity, answerGlobal, type CommunitySummaryRow } from "./indexing/community_summaries.js";
import { ROLE_PERMISSIONS, type AgentRole } from "./access-control.js";
import type {
Store,
MemoryStats,
MemoryLimits,
KbStats,
SearchOptions,
ExplainResult,
BroadcastOptions,
RecallOptions,
ChainStatus,
TokenPayload,
FetchStats,
} from "./store.js";
import type {
MemoryFact,
BroadcastType,
BroadcastMessage,
BroadcastResult,
KnowledgeEntry,
CrossProjectEntry,
RetentionTier,
ComplexityProfile,
CallImpactResult,
CallImpactTarget,
} from "./store.js";
import { projectHash as scopedProjectHash, todayUtc } from "./store.js";
const { Pool } = pg;
// ─────────────────────────────────────────────────────────────────────────────
// Internal helpers
// ─────────────────────────────────────────────────────────────────────────────
function ph(projectPath: string): string {
return scopedProjectHash(projectPath);
}
/** Lever-4 diversity guard: `event:` pseudo-entries are one-liners, so BM25's
* length normalization over-ranks them and they crowd real content out of
* top-K (measured on the T5c bench: multi-session dropped 13 pts). Cap them
* per result set; freed slots fill with the next-best non-event candidates.
* ZC_EVENT_RESULT_CAP overrides (default 3). */
export function capEventEntries<T>(ranked: T[], limit: number, src: (x: T) => string): T[] {
const cap = Math.max(0, parseInt(process.env["ZC_EVENT_RESULT_CAP"] || "3", 10));
const out: T[] = [];
let ev = 0;
for (const r of ranked) {
if (out.length >= limit) break;
if (src(r).startsWith("event:")) {
if (ev >= cap) continue;
ev++;
}
out.push(r);
}
return out;
}
function sanitize(s: string, max: number): string {
return String(s).replace(/[\r\n\x00\x01-\x08\x0b\x0c\x0e-\x1f]/g, " ").trim().slice(0, max);
}
// Scrypt helpers (identical parameters to SqliteStore / memory.ts)
// Token helpers (identical algorithm to access-control.ts)
function getOrCreateSigningKey(pool: pg.Pool, projectHash: string): Promise<string> {
return pool.query<{ value: string }>(
"SELECT value FROM project_meta WHERE project_hash = $1 AND key = 'zc_token_signing_key'",
[projectHash]
).then(async (res) => {
if (res.rows.length > 0) return res.rows[0]!.value;
const newKey = randomBytes(32).toString("hex");
await pool.query(
"INSERT INTO project_meta(project_hash, key, value) VALUES ($1, 'zc_token_signing_key', $2) ON CONFLICT DO NOTHING",
[projectHash, newKey]
);
// Re-read in case of race
const res2 = await pool.query<{ value: string }>(
"SELECT value FROM project_meta WHERE project_hash = $1 AND key = 'zc_token_signing_key'",
[projectHash]
);
return res2.rows[0]!.value;
});
}
function hmacSign(payload: string, key: string): string {
return createHmac("sha256", key).update(payload).digest("hex");
}
// ─────────────────────────────────────────────────────────────────────────────
// PostgresStore
// ─────────────────────────────────────────────────────────────────────────────
export class PostgresStore implements Store {
private pool: pg.Pool;
constructor(connectionString: string) {
this.pool = new Pool({
connectionString,
max: parseInt(process.env["ZC_PG_POOL_SIZE"] ?? "10", 10),
idleTimeoutMillis: 30_000,
connectionTimeoutMillis: 5_000,
// Statement timeout: 30s — prevents runaway queries
options: "--statement_timeout=30000",
});
}
/**
* Run on first use. Verifies the connection and applies all schema migrations.
* Idempotent — safe to call multiple times.
*/
async init(): Promise<void> {
const client = await this.pool.connect();
try {
// Verify pgvector is available
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
// Apply all schema DDL (idempotent — uses IF NOT EXISTS / DO NOTHING)
await client.query(PG_SCHEMA_DDL);
} finally {
client.release();
}
}
// ── Working Memory ────────────────────────────────────────────────────────
async remember(projectPath: string, key: string, value: string, importance: number, agentId: string, epi: EpistemicOpts = {}): Promise<VerifyResult | void> {
const projectHash = ph(projectPath);
const safeKey = sanitize(key, 100);
// v0.52.0 — a clamp that the caller cannot detect is a silent failure, which
// effect verification now flags on every write. Announce it instead: the
// detector immediately caught this one on its first live run (900 chars in,
// 500 stored, {ok:true} out, 400 chars gone with no trace).
const clamped500 = clampWithMarker(sanitize(value, 100_000), Math.max(500, epi.valueMax ?? 0), "fact value");
const safeImp = Math.max(1, Math.min(5, Math.round(importance)));
const safeAgent = sanitize(agentId, 64);
const now = new Date().toISOString();
// v0.31.0 epistemology — explicit kind wins, else auto-classify (parity with rememberFact).
const KINDS: readonly string[] = MEMORY_KINDS; // single source of truth — see memory.ts
const RES = ["open", "resolved_correct", "resolved_incorrect", "resolved_partial"];
const safeKind = epi.kind && KINDS.includes(epi.kind) ? epi.kind : classifyFactKind(clamped500);
// v0.51.2 — pinned kinds get a longer value budget than the flat 500 chars.
// Reported by an agent reading its own recall: every pinned rule was cut
// mid-word at exactly the actionable clause ("HOW TO A…", "assigns QA the
// lit…"). A constraint truncated before it says what to DO is decoration.
// Non-pinned facts keep the 500-char clamp byte-for-byte, and the kind is
// still classified from the 500-char text so classification is unchanged.
// The pinned path must clamp WITH THE MARKER, exactly as the 500 path does.
// Found by a live agent test 2026-08-12: a 2504-char constraint lost 505
// characters with no TRUNCATED marker, so effect-verification reported the
// write as FAILED — correctly, because silent loss is indistinguishable from
// a broken write. v0.51.2 raised the pinned budget and dropped this path off
// clampWithMarker at the same time; the longer budget was the point, losing
// the marker was not.
const safeValue = isPinnedKind({ key: safeKey, importance: safeImp, kind: safeKind })
? clampWithMarker(sanitize(value, 100_000), Math.max(500, Config.PINNED_VALUE_MAX), "fact value")
: clamped500;
const safeConf = (typeof epi.confidence === "number" && isFinite(epi.confidence)) ? Math.max(0, Math.min(1, epi.confidence)) : null;
const safeRes = epi.resolution && RES.includes(epi.resolution) ? epi.resolution : null;
const resolvedAt = (safeRes && safeRes !== "open") ? now : null;
// R1 — optional TTL: validate ISO, must be in the future; invalid values dropped.
// v0.51.3 — per-task markers get a DEFAULT TTL when the writer omits one.
//
// Measured on the live A2A project: 97 live OWNERSHIP_*/ACCEPTANCE_* markers
// consumed 52,982 chars — more than 3x the entire recall budget — and 29 of
// them carried no expiry at all. The convention "per-task notes must set
// ttl_days" was documented but unenforced, so it decayed to a suggestion.
// The orchestrator, asked what was actually crowding out its recall, named
// these markers rather than the pinned rules I suspected.
//
// Only convention-named, non-pinned, importance<=4 facts are affected, so a
// durable decision or constraint can never be silently expired.
// ZC_TASK_MARKER_TTL_DAYS=0 disables (previous behaviour, byte-identical).
const looksPerTask = /^(OWNERSHIP|ACCEPTANCE|ACCEPT|TASK|CKPT|CLAIM)[_-]/i.test(safeKey);
const autoTtlDays = Config.TASK_MARKER_TTL_DAYS;
const wantsAutoTtl =
!epi.expiresAt && autoTtlDays > 0 && looksPerTask && safeImp <= 4 &&
!isPinnedKind({ key: safeKey, importance: safeImp, kind: safeKind });
const safeExpires: string | null = (() => {
if (wantsAutoTtl) return new Date(Date.now() + autoTtlDays * 864e5).toISOString();
if (!epi.expiresAt) return null;
const t = Date.parse(String(epi.expiresAt));
return Number.isFinite(t) && t > Date.now() ? new Date(t).toISOString() : null;
})();
const _ins = await this.pool.query(`
INSERT INTO working_memory(project_hash, key, value, importance, agent_id, created_at, kind, confidence, resolution_status, resolved_at, origin, expires_at, created_by)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
ON CONFLICT(project_hash, key, agent_id) DO UPDATE SET
value = EXCLUDED.value,
importance = EXCLUDED.importance,
created_at = EXCLUDED.created_at,
kind = EXCLUDED.kind,
confidence = EXCLUDED.confidence,
resolution_status = EXCLUDED.resolution_status,
resolved_at = EXCLUDED.resolved_at,
origin = EXCLUDED.origin,
expires_at = EXCLUDED.expires_at,
created_by = COALESCE(EXCLUDED.created_by, working_memory.created_by),
valid_to = NULL,
superseded_by = NULL,
retired_reason = NULL
RETURNING key, value, importance, kind, agent_id
`, [projectHash, safeKey, safeValue, safeImp, safeAgent, now, safeKind, safeConf, safeRes, resolvedAt, epi.origin ? sanitize(epi.origin, 120) : "zc_remember", safeExpires, epi.createdBy ? sanitize(epi.createdBy, 64) : null]);
// (valid_to reset: re-asserting a RETIRED key REVIVES it — the agent explicitly said it again.)
// v0.52.0 — EFFECT VERIFICATION. Compare what the database actually stored
// against what the caller asked for. This is the detector that would have
// caught the kind:'constraint' -> 'fact' coercion on the day it shipped,
// instead of three live E2E rounds later. `kind` is 'exact' precisely
// because silently changing it is the bug; `value` is 'lossy-marked' so a
// clamp must announce itself.
let verification: VerifyResult | undefined;
if (Config.EFFECT_VERIFY) {
const stored = _ins.rows?.[0] ?? {};
verification = verifyWrite(
{ key: safeKey, value, importance: safeImp, kind: safeKind, agent_id: safeAgent },
stored as Record<string, unknown>,
{ key: "exact", kind: "exact", importance: "exact", agent_id: "exact", value: "lossy-marked" },
{ operation: "zc_remember" }
);
if (!verification.ok && Config.EFFECT_VERIFY_STRICT) {
throw new Error(`effect verification failed — ${verification.notice}`);
}
}
// v0.36.0 — memory facts are now co-reference sources, so a memory WRITE must refresh
// the backlink graph too (previously only indexing did — memory edges would go stale).
// Debounced 5s + fire-and-forget: a burst of remembers still costs one rebuild.
this._scheduleBacklinkRebuild(projectPath);
// S1 (v0.44.0) — WRITE-TIME embedding (PG parity with memory.ts). Found during the
// S1 bench: the PG path relied entirely on the 30-min enrichment cron backfill
// (40 facts/cycle), so focused recall ran with rel=0 on every fact for up to an
// hour after a write burst. Fire-and-forget; the cron backfill remains the healer
// for anything dropped here (Ollama down, transient failure).
this._embedFactAsync(projectHash, safeAgent, safeKey, safeValue);
// M1 (v0.41.0) — embed the LIVE fact (fire-and-forget, content-hash deduped) so
// focused recall can rank it by relevance. Same memory:<agent>:<key> source the
// eviction archive uses.
void this._storeEmbedding(projectHash, safeValue, `memory:${safeAgent}:${safeKey}`);
// Evict if over the dynamic limit
const limits = await this.getWorkingMemoryLimits(projectPath);
const countRes = await this.pool.query<{ n: string }>(
"SELECT COUNT(*) as n FROM working_memory WHERE project_hash = $1 AND agent_id = $2 AND valid_to IS NULL",
[projectHash, safeAgent]
);
const count = parseInt(countRes.rows[0]!.n, 10);
if (count > limits.max) {
const toEvictCount = count - limits.evictTo;
// v0.31.0: protect explicitly-tracked OPEN predictions/hypotheses + high-confidence decisions
// (additive — plain facts match neither clause and evict exactly as before).
const PROTECT = `NOT (
(kind IN ('prediction','hypothesis') AND resolution_status = 'open')
OR (kind = 'decision' AND confidence IS NOT NULL AND confidence >= 0.8)
)`;
const victims = (await this.pool.query<{ key: string; value: string }>(
`SELECT key, value FROM working_memory
WHERE project_hash = $1 AND agent_id = $2 AND valid_to IS NULL AND ${PROTECT}
ORDER BY importance ASC, created_at ASC
LIMIT $3`,
[projectHash, safeAgent, toEvictCount]
)).rows;
// Safety valve: if protected facts leave us short, fall back to unfiltered eviction
// for the remainder so the hard `max` bound always holds.
if (victims.length < toEvictCount) {
const have = new Set(victims.map((v) => v.key));
const extra = (await this.pool.query<{ key: string; value: string }>(
`SELECT key, value FROM working_memory
WHERE project_hash = $1 AND agent_id = $2 AND valid_to IS NULL
ORDER BY importance ASC, created_at ASC
LIMIT $3`,
[projectHash, safeAgent, toEvictCount]
)).rows;
for (const r of extra) { if (victims.length >= toEvictCount) break; if (!have.has(r.key)) { victims.push(r); have.add(r.key); } }
}
for (const row of victims) {
await this.pool.query(
"DELETE FROM working_memory WHERE project_hash = $1 AND key = $2 AND agent_id = $3",
[projectHash, row.key, safeAgent]
);
// Archive evicted fact to KB
await this.index(projectPath, row.value, `memory:${safeAgent}:${row.key}`);
}
}
// Surfaced to the caller so the AGENT sees it — a discrepancy that only
// reaches a log is still a silent failure from the agent's point of view.
return verification;
}
/**
* v0.52.1 — per-agent liveness signal for the dispatcher's turn-death detector.
*
* A live agent that receives input ALWAYS does something observable: it calls a
* zc_* tool or it broadcasts. An agent whose turn died on a transient API error
* (529 Overloaded, rate limit, network blip) does neither, while its process
* stays alive and idle — indistinguishable from "thinking" unless you can see
* that it has produced nothing since the moment it was last spoken to.
*
* Measured cost of not having this: a 529 parked three agents for 2.5 hours.
* Nothing detected it; the dispatcher's idle heuristic mislabelled it as a
* stuck worker and escalated a false alarm to the wrong agent.
*/
async agentActivity(projectPath: string): Promise<Array<{
agent_id: string; last_tool_call: string | null; last_broadcast: string | null; last_any: string | null;
}>> {
const projectHash = ph(projectPath);
const res = await this.pool.query<{
agent_id: string; last_tool_call: string | null; last_broadcast: string | null;
}>(`
WITH t AS (
SELECT agent_id, MAX(ts) AS last_tool_call
FROM tool_calls_pg WHERE project_hash = $1 GROUP BY agent_id
), b AS (
SELECT agent_id, MAX(created_at::timestamptz) AS last_broadcast
FROM broadcasts WHERE project_hash = $1 GROUP BY agent_id
)
SELECT COALESCE(t.agent_id, b.agent_id) AS agent_id,
t.last_tool_call::text,
b.last_broadcast::text
FROM t FULL OUTER JOIN b ON t.agent_id = b.agent_id
WHERE COALESCE(t.agent_id, b.agent_id) IS NOT NULL
`, [projectHash]);
return res.rows.map((r) => {
const times = [r.last_tool_call, r.last_broadcast]
.filter(Boolean).map((x) => Date.parse(String(x))).filter(Number.isFinite);
return {
agent_id: r.agent_id,
last_tool_call: r.last_tool_call,
last_broadcast: r.last_broadcast,
last_any: times.length ? new Date(Math.max(...times)).toISOString() : null,
};
});
}
// ── v0.37.0 Temporal fact retirement ───────────────────────────────────────
async retireFact(projectPath: string, key: string, agentId: string, supersededBy: string | null, reason: string): Promise<boolean> {
return this._retireFactByHash(ph(projectPath), key, agentId, supersededBy, reason);
}
private async _retireFactByHash(projectHash: string, key: string, agentId: string, supersededBy: string | null, reason: string): Promise<boolean> {
const safeKey = sanitize(key, 100);
const safeAgent = sanitize(agentId, 64);
const row = (await this.pool.query<{ value: string; kind: string | null }>(
"SELECT value, kind FROM working_memory WHERE project_hash = $1 AND key = $2 AND agent_id = $3 AND valid_to IS NULL",
[projectHash, safeKey, safeAgent])).rows[0];
if (!row) return false;
// v0.51.2 — AUTOMATIC retirement can never remove a pinned kind.
//
// Caught by dogfooding on the live A2A project: four standing operator rules
// typed as 'constraint' were auto-retired with reason 'superseded'. Two of
// them lost to `last_session_summary`, and one rule was killed by a DIFFERENT
// rule. The contradiction adjudicator picks the survivor by recency, so a June
// constraint always loses to a July note that merely embeds near it.
//
// That is the same incident this feature exists to prevent, arriving through
// another door: the pinned tier stops the BUDGET from hiding a constraint, but
// supersession deleted it from recall outright — and silently, since retirement
// leaves the fact findable by zc_search, so it looks present while being absent
// from every recall. An operator rule may only be retired by an operator.
//
// Explicit retirement (zc_forget, operator dashboard) passes its own reason and
// is unaffected. ZC_PIN_CONSTRAINTS=0 restores the previous behaviour.
const AUTOMATIC_REASONS = new Set(["superseded", "consolidated", "expired"]);
if (AUTOMATIC_REASONS.has(reason) && isPinnedKind({ key: safeKey, importance: 0, kind: row.kind })) {
const { logger } = await import("./logger.js");
logger.info("memory", "pinned_retire_refused", {
project_hash: projectHash, agent_id: safeAgent, key: safeKey,
kind: row.kind, reason, superseded_by: supersededBy,
});
return false;
}
await this.pool.query(
"UPDATE working_memory SET valid_to = NOW(), superseded_by = $4, retired_reason = $5 WHERE project_hash = $1 AND key = $2 AND agent_id = $3",
[projectHash, safeKey, safeAgent, supersededBy ? sanitize(supersededBy, 100) : null, sanitize(reason, 100)]);
// Archive to the KB by hash (mirrors index()'s upserts — retire is non-destructive:
// the value stays findable via zc_search and revivable via reviveFact).
const source = `memory:${safeAgent}:${safeKey}`;
const now = new Date().toISOString();
try {
await this.pool.query(`
INSERT INTO knowledge_entries(project_hash, source, content, created_at, first_seen_at, last_indexed_at)
VALUES ($1, $2, $3, $4, NOW(), NOW())
ON CONFLICT(project_hash, source) DO UPDATE SET content = EXCLUDED.content, created_at = EXCLUDED.created_at,
first_seen_at = COALESCE(knowledge_entries.first_seen_at, EXCLUDED.first_seen_at), last_indexed_at = NOW()
`, [projectHash, source, row.value, now]);
await this.pool.query(`
INSERT INTO source_meta(project_hash, source, source_type, retention_tier, created_at, l0_summary, l1_summary)
VALUES ($1, $2, 'internal', 'internal', $3, $4, $5)
ON CONFLICT(project_hash, source) DO UPDATE SET created_at = EXCLUDED.created_at, l0_summary = EXCLUDED.l0_summary, l1_summary = EXCLUDED.l1_summary
`, [projectHash, source, now, row.value.slice(0, Config.TIER_L0_CHARS).trim(), row.value.slice(0, Config.TIER_L1_CHARS).trim()]);
} catch { /* archival is best-effort — retirement itself already succeeded */ }
void this._rebuildBacklinksByHash(projectHash).catch(() => undefined);
return true;
}
async reviveFact(projectPath: string, key: string, agentId: string): Promise<boolean> {
return this._reviveFactByHash(ph(projectPath), key, agentId);
}
private async _reviveFactByHash(projectHash: string, key: string, agentId: string): Promise<boolean> {
const r = await this.pool.query(
"UPDATE working_memory SET valid_to = NULL, superseded_by = NULL, retired_reason = NULL WHERE project_hash = $1 AND key = $2 AND agent_id = $3 AND valid_to IS NOT NULL",
[projectHash, sanitize(key, 100), sanitize(agentId, 64)]);
if ((r.rowCount ?? 0) > 0) { void this._rebuildBacklinksByHash(projectHash).catch(() => undefined); return true; }
return false;
}
async forget(projectPath: string, key: string, agentId: string): Promise<boolean> {
const projectHash = ph(projectPath);
const safeKey = sanitize(key, 100);
const safeAgent = sanitize(agentId, 64);
// v0.38.0 — SOFT DELETE with a recovery window: forget RETIRES the fact (out of recall
// immediately, KB-archived, revivable for RETIRE_PURGE_DAYS) instead of hard-deleting.
void projectHash;
return this.retireFact(projectPath, safeKey, safeAgent, null, "forgotten");
}
async recall(
projectPath: string,
agentId: string,
opts: { focus?: string; from?: Date; to?: Date; asOf?: Date; role?: string } = {},
): Promise<MemoryFact[]> {
const projectHash = ph(projectPath);
const safeAgent = sanitize(agentId, 64);
// M3 (v0.41.0) — AS-OF time travel: reconstruct what was true at a past moment.
// Includes facts retired SINCE then (they were live at asOf) and excludes facts
// created after — the transaction timeline (created_at/valid_to) makes this a
// pure predicate change, no history table needed. (Applied in the branch SQL below.)
// v0.22.2 — per-agent namespacing with shared pool. Each agent gets its
// own private notebook (agent_id = ZC_AGENT_ID = "developer", "orchestrator",
// etc.) AND always sees the project-wide "default" pool (cross-agent
// coordination: ownership tracking, last_session_summary, project state).
//
// Why: previously every fact was written under "default" and recall
// returned all 101+ facts for any agent on any task — massive token
// overhead for "tiny work." Per-agent gives each agent ONLY their own
// private decisions + the shared coordination layer.
//
// When agentId="default" explicitly: return only the shared pool
// (avoids redundant self-join).
let rows: MemoryFact[];
const COLS = `key, value, importance, agent_id, created_at, kind, confidence, resolution_status, resolved_at, access_count, last_retrieved_at, origin, valid_at, created_by`;
// S1 (v0.44.0) — historical WINDOW queries include RETIRED facts whose
// event-time falls inside the window. Once auto-supersession retires a stale
// fact, "what did we decide three weeks ago, before the change?" must still
// surface it — it was the truth THEN (Zep invalid_at semantics: superseded,
// not erased). Live-only remains the rule for unwindowed recall.
const windowClause = (base: number): { sql: string; params: unknown[] } => {
const parts: string[] = [];
const params: unknown[] = [];
let n = base;
if (opts.from) { params.push(opts.from); parts.push(`COALESCE(valid_at::timestamptz, created_at::timestamptz) >= $${n++}`); }
if (opts.to) { params.push(opts.to); parts.push(`COALESCE(valid_at::timestamptz, created_at::timestamptz) <= $${n++}`); }
return { sql: `(valid_to IS NOT NULL AND ${parts.join(" AND ")})`, params };
};
if (safeAgent === "default") {
// R1 — expired facts are excluded from live recall (the sweep formally retires them).
let live = opts.asOf ? `created_at <= $2 AND (valid_to IS NULL OR valid_to > $2)` : `valid_to IS NULL AND (expires_at IS NULL OR expires_at > NOW())`;
const params: unknown[] = opts.asOf ? [projectHash, opts.asOf] : [projectHash];
if (!opts.asOf && (opts.from || opts.to)) {
const w = windowClause(params.length + 1);
live = `(${live} OR ${w.sql})`;
params.push(...w.params);
}
const res = await this.pool.query<MemoryFact>(
`SELECT ${COLS}
FROM working_memory WHERE project_hash = $1 AND agent_id = 'default' AND ${live}
ORDER BY importance DESC, created_at DESC`,
params
);
rows = res.rows;
} else {
// For per-agent agentId: UNION (their private notebook) + (shared 'default' pool)
let live = opts.asOf ? `created_at <= $3 AND (valid_to IS NULL OR valid_to > $3)` : `valid_to IS NULL AND (expires_at IS NULL OR expires_at > NOW())`;
const params: unknown[] = opts.asOf ? [projectHash, safeAgent, opts.asOf] : [projectHash, safeAgent];
if (!opts.asOf && (opts.from || opts.to)) {
const w = windowClause(params.length + 1);
live = `(${live} OR ${w.sql})`;
params.push(...w.params);
}
const res = await this.pool.query<MemoryFact>(
`SELECT ${COLS}
FROM working_memory
WHERE project_hash = $1 AND (agent_id = $2 OR agent_id = 'default') AND ${live}
ORDER BY
CASE WHEN agent_id = $2 THEN 0 ELSE 1 END,
importance DESC,
created_at DESC`,
params
);
rows = res.rows;
}
// v0.54.0 - CROSS-PROJECT pinned lessons. An antipattern about how code fails
// ("a stub returning a benign default hides a missing implementation") is not
// about one repo. Measured: the identical class hit SecureContext and A2A
// hours apart with nothing connecting them, because memory is per-project.
//
// Only PINNED kinds cross the boundary, and only from the reserved global
// scope an author opts into - project facts never leak sideways.
if (Config.SHARE_GLOBAL_PINNED && !opts.asOf) {
try {
const g = await this.pool.query<MemoryFact>(
`SELECT ${COLS} FROM working_memory
WHERE project_hash = $1 AND valid_to IS NULL
AND kind IN ('constraint','antipattern')
ORDER BY importance DESC, created_at DESC
LIMIT 12`,
// ph() the sentinel: remember() hashes whatever projectPath it is given,
// so the write lands under hash("__global__"). Matching the raw literal
// here made the pool unreachable - a write that succeeds into something
// nothing reads. Both sides must hash identically.
[ph(Config.GLOBAL_PROJECT_HASH)]);
const seen = new Set(rows.map((r) => r.key));
for (const gr of g.rows) if (!seen.has(gr.key)) rows.push(gr);
} catch { /* global scope is additive; never break recall */ }
}
// Tier-2 #4: secondary salience re-sort (importance stays primary) + best-effort
// access bump (single batched UPDATE via unnest, fire-and-forget). Inert when
// W_SALIENCE=0 — byte-identical ordering, no writes (the kill-switch).
// R8 (v0.43.0): sort key is EFFECTIVE importance (staleness-demoted, see
// recall_budget.ts; inert when ZC_RECALL_STALE_DEMOTE=0) and the bump covers
// only the facts that will RENDER under the recall budget — bumping every row
// reset last_retrieved_at project-wide each recall, making "stale" undetectable.
const demoteStale = Config.RECALL_STALE_DEMOTE > 0;
if ((salienceEnabled() || demoteStale) && rows.length > 0) {
const now = Date.now();
const k = (r: MemoryFact) => `${r.key} ${r.agent_id ?? ""}`;
const sal = salienceEnabled()
? new Map(rows.map((r) => [k(r), computeSalience(r.access_count, r.last_retrieved_at ?? null, now)]))
: null;
const prio = (r: MemoryFact) => (safeAgent !== "default" && r.agent_id === safeAgent ? 0 : 1);
const eff = (r: MemoryFact) => (demoteStale ? effectiveImportance(r, now) : r.importance);
// v0.54.0 - role affinity RANKS, it does not filter. QA carrying the
// developer's private constraints is noise worth demoting; hiding a fact
// from a role that turns out to need it is the silent-loss failure this
// codebase spent a day removing. Pinned kinds are exempt - a standing rule
// applies to everyone regardless of who wrote it.
const wRole = Config.W_ROLE_AFFINITY;
const callerRole = String(opts.role ?? "").trim().toLowerCase();
const roleBoost = (r: MemoryFact): number => {
if (!wRole || !callerRole) return 0;
if (["constraint", "antipattern"].includes(String(r.kind ?? ""))) return 0;
const owner = String(r.agent_id ?? "").toLowerCase();
if (owner === callerRole) return wRole; // mine
if (owner === "default") return wRole / 2; // shared, applies to all
return -wRole; // another role's private note
};
rows = [...rows].sort((a, b) =>
prio(a) - prio(b) ||
(eff(b) + roleBoost(b)) - (eff(a) + roleBoost(a)) ||
(sal ? (sal.get(k(b)) ?? 0) - (sal.get(k(a)) ?? 0) : 0) ||
(a.created_at < b.created_at ? 1 : a.created_at > b.created_at ? -1 : 0)
);
if (salienceEnabled()) {
const toBump = budgetFacts(rows).rendered;
void this.pool.query(
`UPDATE working_memory AS w
SET access_count = COALESCE(w.access_count,0) + 1, last_retrieved_at = NOW()
FROM unnest($2::text[], $3::text[]) AS t(key, agent_id)
WHERE w.project_hash = $1 AND w.key = t.key AND w.agent_id = t.agent_id`,
[projectHash, toBump.map((r) => r.key), toBump.map((r) => r.agent_id ?? safeAgent)]
).catch(() => undefined);
}
}
// M1 (v0.41.0) — FOCUSED recall: with a focus string, re-rank live facts by
// blended relevance to the agent's CURRENT task (the M0 benchmark showed
// task-relevant facts ranking 74-79/81 under importance-only ordering).
// score = RECALL_W_REL·cosine + RECALL_W_IMP·(importance/5) + RECALL_W_SAL·salience
// Missing vectors ⇒ rel=0 (importance still ranks them — graceful until the
// backfill lands). Ollama down ⇒ unfocused order unchanged. No focus ⇒ byte-identical.
if (opts.focus && opts.focus.trim() && rows.length > 0) {
try {
const qEmbed = await getEmbedding(opts.focus.slice(0, 2000));
if (qEmbed) {
const sources = rows.map((r) => `memory:${r.agent_id ?? safeAgent}:${r.key}`);
const embRes = await this.pool.query<{ source: string; vector: string }>(
`SELECT source, vector::text FROM embeddings
WHERE project_hash = $1 AND model_name = $2 AND source = ANY($3)`,
[projectHash, ACTIVE_MODEL, sources]
);
const vecMap = new Map(embRes.rows.map((r) => [r.source, r.vector]));
const now = Date.now();
const scoreOf = (r: MemoryFact): number => {
const vs = vecMap.get(`memory:${r.agent_id ?? safeAgent}:${r.key}`);
let rel = 0;
if (vs) {
const nums = vs.slice(1, -1).split(",").map(Number);
rel = Math.max(0, cosineSimilarity(new Float32Array(nums), qEmbed.vector));
}
const sal = computeSalience(r.access_count, r.last_retrieved_at ?? null, now);
let score = Config.RECALL_W_REL * rel + Config.RECALL_W_IMP * (r.importance / 5) + Config.RECALL_W_SAL * sal;
// M3 — temporal window bonus: event-time (valid_at, else created_at)
// inside the parsed window ranks the fact above topic-only matches.
if (opts.from || opts.to) {
const evRaw = (r as MemoryFact & { valid_at?: string | Date | null }).valid_at ?? r.created_at;
const ev = evRaw instanceof Date ? evRaw.getTime() : Date.parse(String(evRaw));
const inWindow =
Number.isFinite(ev) &&
(!opts.from || ev >= opts.from.getTime()) &&
(!opts.to || ev <= opts.to.getTime());
// R3 — measured verdict: on the labeled corpus BOTH a hard relevance
// gate and a proportional bonus scored WORSE than the flat bonus
// (gold/noise relevance ranges overlap). Flat is the default;
// ZC_RECALL_TEMPORAL_REL_GATE>0 re-enables gating for corpora
// where the ranges separate.
if (inWindow) {
score += Config.RECALL_TEMPORAL_REL_GATE > 0
? (rel >= Config.RECALL_TEMPORAL_REL_GATE ? Config.RECALL_W_TEMPORAL : 0)
: Config.RECALL_W_TEMPORAL;
}
}
return score;
};
const scores = new Map(rows.map((r) => [r, scoreOf(r)]));
const bySorted = (a: MemoryFact, b: MemoryFact) =>
(scores.get(b)! - scores.get(a)!) ||
(b.importance - a.importance) ||
(a.created_at < b.created_at ? 1 : a.created_at > b.created_at ? -1 : 0);
rows = [...rows].sort(bySorted);
// S1 (v0.44.0) — prefer-latest (mirrors memory.ts): among the top candidates,
// a near-identical conflicting pair demotes the OLDER fact below the newer.
// Skipped for temporal/as-of queries — historical questions want the old fact.
if (Config.PREFER_LATEST && !opts.from && !opts.to && !opts.asOf && rows.length > 1) {
const { preferLatestAdjust } = await import("./contradiction_heuristics.js");
const parseVec = (r: MemoryFact): Float32Array | undefined => {
const vs = vecMap.get(`memory:${r.agent_id ?? safeAgent}:${r.key}`);
return vs ? new Float32Array(vs.slice(1, -1).split(",").map(Number)) : undefined;
};
const evOf = (r: MemoryFact): number => {
const raw = (r as MemoryFact & { valid_at?: string | Date | null }).valid_at ?? r.created_at;
return raw instanceof Date ? raw.getTime() : Date.parse(String(raw));
};
// Fixpoint loop: demoting a stale duplicate frees top-K slots that can
// expose NEW conflicting pairs (measured: 6 stale worklogs blocked the
// updated cache-TTL fact out of the window, so its stale twin was never
// co-examined). Re-slice and re-run until a pass adjusts nothing (≤3).
for (let pass = 0; pass < 3; pass++) {
const top = rows.slice(0, Config.PREFER_LATEST_TOPK).map((r) => ({
fact: r, score: scores.get(r)!, vec: parseVec(r), ev: evOf(r),
}));
const adjusted = preferLatestAdjust(top, cosineSimilarity, Config.PREFER_LATEST_MARGIN);
if (adjusted.size === 0) break;
for (const r of rows) {
const adj = adjusted.get(r.key);
if (adj !== undefined && adj < scores.get(r)!) scores.set(r, adj);
}
rows = [...rows].sort(bySorted);
}
}
}
} catch { /* focus ranking is best-effort — fall back to unfocused order */ }
}
return rows;
}
async archiveSummary(projectPath: string, summary: string): Promise<{ submitted: number; stored: number; dropped: number }> {
// v0.52.5 - ONE clamp, not two. A live agent measured the gap: 2456 chars
// submitted, 400 kept, a marker naming 1499 lost - and 557 chars that died
// in this upstream sanitize() before the marker logic ever saw them, so the
// marker under-reported the loss. A truncation notice that is itself wrong
// is worse than none: it tells the reader the damage is bounded when it is not.
const safe = clampWithMarker(sanitize(summary, 100_000), Config.BROADCAST_SUMMARY_MAX, "session summary");
const now = new Date().toISOString();
const source = `[SESSION_SUMMARY] ${now.slice(0, 10)}`;
await this.index(projectPath, safe, source, "internal", "summary");
// v0.52.4 - a live agent found this one: zc_summarize_session silently lost
// 1500 chars of session summary and reported "Session summary archived."
// The session summary is what the NEXT session reads to resume, so a silent
// clamp here loses continuity precisely where it matters most.
await this.remember(projectPath, "last_session_summary", safe, 5, "default",
{ valueMax: Config.BROADCAST_SUMMARY_MAX });
return { submitted: summary.length, stored: safe.length, dropped: Math.max(0, summary.length - safe.length) };
}
async getMemoryStats(projectPath: string, agentId: string): Promise<MemoryStats> {
const projectHash = ph(projectPath);
const safeAgent = sanitize(agentId, 64);
const [countRes, critRes] = await Promise.all([
this.pool.query<{ n: string }>(
"SELECT COUNT(*) as n FROM working_memory WHERE project_hash = $1 AND agent_id = $2 AND valid_to IS NULL",
[projectHash, safeAgent]
),
this.pool.query<{ n: string }>(
"SELECT COUNT(*) as n FROM working_memory WHERE project_hash = $1 AND agent_id = $2 AND importance >= 4 AND valid_to IS NULL",
[projectHash, safeAgent]
),
]);
const limits = await this.getWorkingMemoryLimits(projectPath);
return {
count: parseInt(countRes.rows[0]!.n, 10),
max: limits.max,
evictTo: limits.evictTo,
criticalCount: parseInt(critRes.rows[0]!.n, 10),
complexity: limits.profile,
};
}
async countImportance5(projectPath: string, agentId: string): Promise<number> {
const res = await this.pool.query<{ n: string }>(
"SELECT COUNT(*) as n FROM working_memory WHERE project_hash = $1 AND agent_id = $2 AND importance = 5 AND valid_to IS NULL",
[ph(projectPath), sanitize(agentId, 64)]
);
return parseInt(res.rows[0]!.n, 10);
}
async getWorkingMemoryLimits(projectPath: string, forceRecompute = false): Promise<MemoryLimits> {
const projectHash = ph(projectPath);
const WM_CACHE_TTL = 10 * 60 * 1000;
if (!forceRecompute) {
const res = await this.pool.query<{ value: string }>(
"SELECT value FROM project_meta WHERE project_hash = $1 AND key = 'zc_complexity_profile'",
[projectHash]
);
if (res.rows.length > 0) {
try {
const cached = JSON.parse(res.rows[0]!.value) as ComplexityProfile;
const ageMs = Date.now() - new Date(cached.computedAt).getTime();
if (ageMs < WM_CACHE_TTL) {
return { max: cached.computedLimit, evictTo: cached.evictTo, profile: cached };
}
} catch { /* malformed or stale cache row -> fall through and recompute */ }
}
}
// Compute fresh
const [kbRes, bcRes, agRes] = await Promise.all([
this.pool.query<{ n: string }>(
"SELECT COUNT(*) as n FROM source_meta WHERE project_hash = $1", [projectHash]
),
this.pool.query<{ n: string }>(
"SELECT COUNT(*) as n FROM broadcasts WHERE project_hash = $1", [projectHash]
),
this.pool.query<{ n: string }>(
"SELECT COUNT(*) as n FROM agent_sessions WHERE project_hash = $1 AND revoked = 0 AND expires_at > $2",
[projectHash, new Date().toISOString()]
),
]);
const kbEntries = parseInt(kbRes.rows[0]!.n, 10);
const broadcastCount = parseInt(bcRes.rows[0]!.n, 10);
const activeAgents = parseInt(agRes.rows[0]!.n, 10);
const kbBonus = Math.min(Math.floor(kbEntries / 15), 60);
const bcBonus = Math.min(Math.floor(broadcastCount / 30), 40);
const agentBonus = Math.min(activeAgents * 15, 50);
const computedLimit = Math.max(100, Math.min(250, 100 + kbBonus + bcBonus + agentBonus));
const evictTo = Math.floor(computedLimit * 0.80);
const computedAt = new Date().toISOString();
const profile: ComplexityProfile = {
kbEntries, broadcastCount, activeAgents,
computedLimit, evictTo, computedAt,
};
await this.pool.query(`
INSERT INTO project_meta(project_hash, key, value) VALUES ($1, 'zc_complexity_profile', $2)
ON CONFLICT(project_hash, key) DO UPDATE SET value = EXCLUDED.value
`, [projectHash, JSON.stringify(profile)]);
return { max: computedLimit, evictTo, profile };
}
// ── Knowledge Base ─────────────────────────────────────────────────────────
async index(
projectPath: string,
content: string,
source: string,
sourceType: "internal" | "external" = "internal",
retentionTier: RetentionTier = sourceType === "external" ? "external" : "internal"
): Promise<void> {
const projectHash = ph(projectPath);
const now = new Date().toISOString();
const safeSource = sanitize(source, 500);
const safeContent = sanitize(content, 50_000);
// L0/L1 summary tiers (same logic as knowledge.ts)
const l0 = safeContent.slice(0, Config.TIER_L0_CHARS).trim();
const l1 = safeContent.slice(0, Config.TIER_L1_CHARS).trim();
// Upsert knowledge entry.
// TKG-T1 (v0.47.0) — bi-temporal: first_seen_at is IMMUTABLE (kept from the
// existing row on conflict), last_indexed_at always bumps. created_at keeps
// its historical bump-on-reindex behavior for backward compat; new temporal
// features read the two explicit columns instead.
await this.pool.query(`
INSERT INTO knowledge_entries(project_hash, source, content, created_at, first_seen_at, last_indexed_at)
VALUES ($1, $2, $3, $4, NOW(), NOW())
ON CONFLICT(project_hash, source) DO UPDATE SET
content = EXCLUDED.content,
created_at = EXCLUDED.created_at,
first_seen_at = COALESCE(knowledge_entries.first_seen_at, EXCLUDED.first_seen_at),
last_indexed_at = NOW()
`, [projectHash, safeSource, safeContent, now]);
// Upsert source_meta
await this.pool.query(`
INSERT INTO source_meta(project_hash, source, source_type, retention_tier, created_at, l0_summary, l1_summary)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT(project_hash, source) DO UPDATE SET
source_type = EXCLUDED.source_type,
retention_tier = EXCLUDED.retention_tier,
created_at = EXCLUDED.created_at,
l0_summary = EXCLUDED.l0_summary,
l1_summary = EXCLUDED.l1_summary
`, [projectHash, safeSource, sourceType, retentionTier, now, l0, l1]);
// Fire-and-forget embedding computation
void this._storeEmbedding(projectHash, safeContent, safeSource);
// Tier-1 A: schedule a debounced backlink-graph rebuild over PG (fire-and-forget).
// THIS is what makes the backlink boost actually fire in the live PG deployment —
// PostgresStore.index does not route through indexContent's SQLite trigger.
this._scheduleBacklinkRebuild(projectPath);
// Lever-4 (v0.48.0): event-fact extraction for session-tier sources (PG
// parity — PostgresStore.index does not route through indexContent).
scheduleEventExtraction(safeContent, safeSource, async (evSource, evContent) => {
await this.index(projectPath, evContent, evSource, sourceType, retentionTier);
});
}
private async _storeEmbedding(projectHash: string, content: string, source: string): Promise<boolean> {
try {
// v0.39.0 — content-addressable dedup (SQLite parity): identical content + same model
// ⇒ skip the Ollama call; hash-match with a DIFFERENT model ⇒ explicit re-embed.
const contentHash = createHash("sha256").update(content).digest("hex");
try {
const existing = (await this.pool.query<{ content_hash: string | null; model_name: string }>(
`SELECT content_hash, model_name FROM embeddings WHERE project_hash = $1 AND source = $2`,
[projectHash, source])).rows[0];
if (existing && existing.content_hash === contentHash && existing.model_name === ACTIVE_MODEL) {
// S9 — the HEAD is deduped, but chunks may not exist yet (content indexed
// before chunking shipped, or a prior chunk pass died mid-way). Ensure
// them; each chunk self-dedups via its own content_hash.
if (Config.EMBED_CHUNKS && !source.startsWith("memory:") && content.length > Config.EMBED_CHUNK_SIZE) {
void this._storeChunkEmbeddings(projectHash, content, source, contentHash).catch(() => undefined);
}
return true;
}
} catch { /* content_hash column absent (pre-migration) — fall through */ }
const result = await getEmbeddingQueued(content); // S1 — background lane
if (!result) {
const now = Date.now();
if (now - PostgresStore._lastEmbedErrLog > 60_000) {
PostgresStore._lastEmbedErrLog = now;
console.error(`[embed] getEmbedding returned null for ${source} (Ollama down/breaker open)`);
}
return false;
}
// pgvector expects "[x1,x2,...,xN]" string format
const vectorStr = "[" + result.vector.join(",") + "]";
await this.pool.query(`
INSERT INTO embeddings(project_hash, source, vector, model_name, dimensions, created_at, content_hash)
VALUES ($1, $2, $3::vector, $4, $5, $6, $7)
ON CONFLICT(project_hash, source) DO UPDATE SET
vector = EXCLUDED.vector,
model_name = EXCLUDED.model_name,
dimensions = EXCLUDED.dimensions,
created_at = EXCLUDED.created_at,
content_hash = EXCLUDED.content_hash
`, [projectHash, source, vectorStr, result.modelName, result.dimensions, new Date().toISOString(), contentHash]);
// S9 (v0.46.0) — chunk embeddings for long content: the head vector only
// covers the first EMBED_MAX_CHARS; store additional per-chunk vectors so
// search can max-pool similarity over the WHOLE document. Chunk rows are
// keyed `<source>#c<N>` (never joined as KB entries; search maps them back
// to the parent). Fire-and-forget per chunk via the background lane.
if (Config.EMBED_CHUNKS && !source.startsWith("memory:") && content.length > Config.EMBED_CHUNK_SIZE) {
void this._storeChunkEmbeddings(projectHash, content, source, contentHash).catch(() => undefined);
}
return true;
} catch (e) {
// Embedding failure is non-fatal — falls back to BM25-only search.
// S1: but never fully silent — five rounds of debugging were spent on a
// pipeline that failed without a single log line. Rate-limited to 1/min.
const now = Date.now();
if (now - PostgresStore._lastEmbedErrLog > 60_000) {
PostgresStore._lastEmbedErrLog = now;
console.error(`[embed] store failed for ${source}: ${(e as Error)?.message || (e as Error)?.name || "unknown"}`);
}
return false;
}
}
private static _lastEmbedErrLog = 0;
/**
* S9 (v0.46.0) — store per-chunk embeddings for content beyond the head window.
* Chunk 0 is the head (already stored under the bare source); chunks start at
* offset EMBED_CHUNK_SIZE with a small overlap so boundary sentences aren't
* split blind. content_hash carries the PARENT hash + chunk index so re-index
* of unchanged content skips cleanly; stale chunks beyond the new count are
* deleted (content shrank).
*/
private async _storeChunkEmbeddings(projectHash: string, content: string, source: string, parentHash: string): Promise<void> {
const size = Math.max(500, Config.EMBED_CHUNK_SIZE);
const overlap = Math.min(300, Math.floor(size / 10));
const chunks: string[] = [];
for (let off = size - overlap; off < content.length && chunks.length < Math.max(1, Config.EMBED_MAX_CHUNKS); off += size - overlap) {
const piece = content.slice(off, off + size);
if (piece.trim().length < 100) break; // tail too small to be a useful vector
chunks.push(piece);
}
for (let i = 0; i < chunks.length; i++) {
const chunkSource = `${source}#c${i + 1}`;
const chunkHash = `${parentHash}:${i + 1}`;
try {
const existing = (await this.pool.query<{ content_hash: string | null; model_name: string }>(
`SELECT content_hash, model_name FROM embeddings WHERE project_hash = $1 AND source = $2`,
[projectHash, chunkSource])).rows[0];
if (existing && existing.content_hash === chunkHash && existing.model_name === ACTIVE_MODEL) continue;
const result = await getEmbeddingQueued(chunks[i]!);
if (!result) return; // embedder down — the backfill cron heals later re-writes
await this.pool.query(`
INSERT INTO embeddings(project_hash, source, vector, model_name, dimensions, created_at, content_hash)
VALUES ($1, $2, $3::vector, $4, $5, $6, $7)
ON CONFLICT(project_hash, source) DO UPDATE SET
vector = EXCLUDED.vector, model_name = EXCLUDED.model_name,
dimensions = EXCLUDED.dimensions, created_at = EXCLUDED.created_at,
content_hash = EXCLUDED.content_hash
`, [projectHash, chunkSource, "[" + result.vector.join(",") + "]", result.modelName, result.dimensions, new Date().toISOString(), chunkHash]);
} catch { /* per-chunk best-effort */ }
}
// Content shrank since last index → drop chunk rows beyond the new count.
try {
await this.pool.query(
`DELETE FROM embeddings WHERE project_hash = $1 AND source LIKE $2
AND CAST(substring(source from '#c([0-9]+)$') AS int) > $3`,
[projectHash, `${source.replace(/([%_\\])/g, "\\$1")}#c%`, chunks.length],
);
} catch { /* best-effort */ }