feat(phase5): hierarchical abstraction engine (v3.11)

package.json

@@ -45,13 +45,14 @@
     "test:dreams-compat": "node --import tsx/esm --test tests/dreams-compat.test.ts",
     "test:dreams-anthropic": "node --import tsx/esm --test tests/dreams-anthropic.test.ts",
     "test:dreams-bridge": "node --import tsx/esm --test tests/dreams-bridge.test.ts",
-    "test": "node --import tsx/esm --test --test-concurrency=1 tests/integration.test.ts tests/llm-paths.test.ts tests/http-api.test.ts tests/cache.test.ts tests/embedding-migration.test.ts tests/outcome-revision.test.ts tests/reflection-revision.test.ts tests/selective-forgetting.test.ts tests/multi-device.test.ts tests/dream-runs.test.ts tests/dreams-compat.test.ts tests/dreams-anthropic.test.ts tests/dreams-bridge.test.ts tests/sentiment-tagging.test.ts tests/adaptive-sleep.test.ts tests/epistemic-confidence.test.ts tests/explainable-memory.test.ts tests/causal-graph.test.ts",
+    "test": "node --import tsx/esm --test --test-concurrency=1 tests/integration.test.ts tests/llm-paths.test.ts tests/http-api.test.ts tests/cache.test.ts tests/embedding-migration.test.ts tests/outcome-revision.test.ts tests/reflection-revision.test.ts tests/selective-forgetting.test.ts tests/multi-device.test.ts tests/dream-runs.test.ts tests/dreams-compat.test.ts tests/dreams-anthropic.test.ts tests/dreams-bridge.test.ts tests/sentiment-tagging.test.ts tests/adaptive-sleep.test.ts tests/epistemic-confidence.test.ts tests/explainable-memory.test.ts tests/causal-graph.test.ts tests/abstractions.test.ts",
     "test:multi-device": "node --import tsx/esm --test tests/multi-device.test.ts",
     "test:sentiment-tagging": "node --import tsx/esm --test tests/sentiment-tagging.test.ts",
     "test:adaptive-sleep": "node --import tsx/esm --test tests/adaptive-sleep.test.ts",
     "test:epistemic-confidence": "node --import tsx/esm --test tests/epistemic-confidence.test.ts",
     "test:explainable-memory": "node --import tsx/esm --test tests/explainable-memory.test.ts",
     "test:causal-graph": "node --import tsx/esm --test tests/causal-graph.test.ts",
+    "test:abstractions": "node --import tsx/esm --test tests/abstractions.test.ts",
     "benchmark:longmemeval": "node --import tsx/esm benchmarks/longmemeval/run.ts",
     "benchmark:download": "node --import tsx/esm benchmarks/longmemeval/download.ts",
     "benchmark:ingest": "node --import tsx/esm benchmarks/longmemeval/ingest.ts",

python/memforge/client.py

@@ -342,6 +342,41 @@ async def predict(
             body["namespace"] = namespace
         return await self._post(f"/memory/{agent_id}/predict", body)
 
+    # ── Hierarchical Abstraction (v3.11) ──────────────────────────────────
+
+    async def get_principles(
+        self,
+        agent_id: str,
+        namespace: str | None = None,
+        limit: int | None = None,
+    ) -> list[dict[str, Any]]:
+        """Active principle-level abstractions (v3.11).
+
+        Cross-cutting principles distilled from meta-reflections by
+        Sleep Phase 5.11, ordered by confidence then recency. The server
+        caps results at 50; limit (1-50) trims further.
+        """
+        params: dict[str, Any] = {"namespace": namespace, "limit": limit}
+        raw = await self._get(f"/memory/{agent_id}/principles", params)
+        return raw if isinstance(raw, list) else []
+
+    async def get_abstractions(
+        self,
+        agent_id: str,
+        level: str | None = None,
+        namespace: str | None = None,
+    ) -> list[dict[str, Any]]:
+        """Active abstractions (v3.11), optionally filtered by level.
+
+        level is 'principle', 'strategy', or 'mental_model'. Ordered by
+        confidence then recency, capped at 50. Sleep Phase 5.11 currently
+        writes only 'principle' rows; 'strategy' and 'mental_model' return
+        [] until something writes them.
+        """
+        params: dict[str, Any] = {"level": level, "namespace": namespace}
+        raw = await self._get(f"/memory/{agent_id}/abstractions", params)
+        return raw if isinstance(raw, list) else []
+
     async def resume(self, agent_id: str, limit: int = 5, namespace: str | None = None) -> ResumeContext:
         """Get session resumption context bundle."""
         params: dict[str, Any] = {"limit": limit}

python/memforge/resilient.py

@@ -166,6 +166,20 @@ async def predict(self, agent_id: str, context: str, namespace: str | None = Non
             self._handle(e)
             return None
 
+    async def get_principles(self, agent_id: str, namespace: str | None = None, limit: int | None = None) -> list[dict[str, Any]]:
+        try:
+            return await self._client.get_principles(agent_id, namespace, limit)
+        except Exception as e:
+            self._handle(e)
+            return []
+
+    async def get_abstractions(self, agent_id: str, level: str | None = None, namespace: str | None = None) -> list[dict[str, Any]]:
+        try:
+            return await self._client.get_abstractions(agent_id, level, namespace)
+        except Exception as e:
+            self._handle(e)
+            return []
+
     async def resume(self, agent_id: str, limit: int = 5, namespace: str | None = None) -> ResumeContext | None:
         try:
             return await self._client.resume(agent_id, limit, namespace)

schema/migration-v3.11.sql

@@ -0,0 +1,60 @@
+-- MemForge — Migration v3.11: Hierarchical Abstraction Engine
+--
+-- Feature 4 of the Phase 5 Autonomous Knowledge Architecture split
+-- (features 1, 5, 6 landed in v3.8/v3.9; features 2, 3 in v3.10).
+--
+-- abstractions stores cross-cutting knowledge distilled from meta-reflections.
+-- Sleep Phase 5.11 (phasePrincipleExtraction) prompts the LLM with the 10 most
+-- recent meta-reflections (reflection_level > 1, minimum 3 required) and
+-- inserts the extracted principles at level 'principle'. Read paths:
+-- getAbstractions() and getPrinciples() (active rows only, ordered by
+-- confidence then recency).
+--
+-- content_hash exists for dedup: principles are re-extracted every sleep
+-- cycle, and identical content must upsert-noop rather than accumulate
+-- duplicate rows. It is a stored generated md5(content), and
+-- UNIQUE (agent_id, level, namespace, content_hash) is the conflict target
+-- for the phase's ON CONFLICT ... DO NOTHING insert.
+--
+-- Apply: psql "$DATABASE_URL" -f schema/migration-v3.11.sql
+
+BEGIN;
+
+-- ─── Feature 4: Hierarchical Abstraction Engine ─────────────────────────────
+
+CREATE TABLE IF NOT EXISTS abstractions (
+  id                    BIGSERIAL   PRIMARY KEY,
+  agent_id              TEXT        NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
+  level                 TEXT        NOT NULL CHECK (level IN ('principle', 'strategy', 'mental_model')),
+  content               TEXT        NOT NULL,
+  content_hash          TEXT        GENERATED ALWAYS AS (md5(content)) STORED,
+  source_reflection_ids BIGINT[]    NOT NULL DEFAULT '{}',
+  confidence            REAL        NOT NULL DEFAULT 0.5,
+  active                BOOLEAN     NOT NULL DEFAULT true,
+  namespace             TEXT        NOT NULL DEFAULT 'default',
+  created_at            TIMESTAMPTZ NOT NULL DEFAULT now(),
+  UNIQUE (agent_id, level, namespace, content_hash)
+);
+
+CREATE INDEX IF NOT EXISTS abstractions_agent_level_idx
+  ON abstractions (agent_id, level, active);
+
+-- ─── RLS on new table ───────────────────────────────────────────────────────
+
+ALTER TABLE abstractions ENABLE ROW LEVEL SECURITY;
+DROP POLICY IF EXISTS abstractions_agent_isolation ON abstractions;
+CREATE POLICY abstractions_agent_isolation ON abstractions
+  FOR ALL
+  USING (agent_id = current_setting('app.current_agent_id', true))
+  WITH CHECK (agent_id = current_setting('app.current_agent_id', true));
+
+-- ─── Grants for memforge_app role (if exists) ───────────────────────────────
+
+DO $$ BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'memforge_app') THEN
+    EXECUTE 'GRANT ALL ON abstractions TO memforge_app';
+    EXECUTE 'GRANT USAGE, SELECT ON SEQUENCE abstractions_id_seq TO memforge_app';
+  END IF;
+END $$;
+
+COMMIT;

schema/schema.sql

@@ -722,6 +722,30 @@ CREATE INDEX IF NOT EXISTS causal_edges_agent_cause_idx
 CREATE INDEX IF NOT EXISTS causal_edges_agent_effect_idx
   ON causal_edges (agent_id, effect_id);
 
+-- ─────────────────────────────────────────────────────────────────────────────
+-- abstractions (v3.11+) — cross-cutting principles distilled from
+-- meta-reflections by Sleep Phase 5.11. content_hash is a stored md5(content);
+-- with UNIQUE (agent_id, level, namespace, content_hash) it makes per-cycle
+-- re-extraction of the same principle an insert no-op instead of a duplicate
+-- row. Read via getAbstractions()/getPrinciples().
+-- ─────────────────────────────────────────────────────────────────────────────
+CREATE TABLE IF NOT EXISTS abstractions (
+  id                    BIGSERIAL   PRIMARY KEY,
+  agent_id              TEXT        NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
+  level                 TEXT        NOT NULL CHECK (level IN ('principle', 'strategy', 'mental_model')),
+  content               TEXT        NOT NULL,
+  content_hash          TEXT        GENERATED ALWAYS AS (md5(content)) STORED,
+  source_reflection_ids BIGINT[]    NOT NULL DEFAULT '{}',
+  confidence            REAL        NOT NULL DEFAULT 0.5,
+  active                BOOLEAN     NOT NULL DEFAULT true,
+  namespace             TEXT        NOT NULL DEFAULT 'default',
+  created_at            TIMESTAMPTZ NOT NULL DEFAULT now(),
+  UNIQUE (agent_id, level, namespace, content_hash)
+);
+
+CREATE INDEX IF NOT EXISTS abstractions_agent_level_idx
+  ON abstractions (agent_id, level, active);
+
 -- ─────────────────────────────────────────────────────────────────────────────
 -- Row-Level Security (v3.0+ fresh installs — backported from migration-v2.3)
 -- FORCE ROW LEVEL SECURITY is intentionally omitted on all tables.
@@ -905,6 +929,13 @@ CREATE POLICY causal_edges_agent_isolation ON causal_edges
   USING (agent_id = current_setting('app.current_agent_id', true))
   WITH CHECK (agent_id = current_setting('app.current_agent_id', true));
 
+ALTER TABLE abstractions ENABLE ROW LEVEL SECURITY;
+DROP POLICY IF EXISTS abstractions_agent_isolation ON abstractions;
+CREATE POLICY abstractions_agent_isolation ON abstractions
+  FOR ALL
+  USING (agent_id = current_setting('app.current_agent_id', true))
+  WITH CHECK (agent_id = current_setting('app.current_agent_id', true));
+
 -- ─────────────────────────────────────────────────────────────────────────────
 -- Service role that bypasses RLS
 -- ─────────────────────────────────────────────────────────────────────────────

src/app.ts

@@ -10,15 +10,15 @@ import type { Request, Response, NextFunction } from 'express';
 import type { MemoryManager } from './memory-manager.js';
 import type { AuditChain } from './audit.js';
 import type { ClassifierRegistry } from './classifier.js';
-import type { QueryMode, ConsolidationMode, FeedbackOutcome } from './types.js';
+import type { QueryMode, ConsolidationMode, FeedbackOutcome, AbstractionLevel } from './types.js';
 import { getLogger, requestIdMiddleware, requestLogMiddleware } from './logger.js';
 import {
   registry,
   httpRequestsTotal,
   httpRequestDurationSeconds,
 } from './metrics.js';
 import { bearerAuth, requireScope, getClientId } from './auth.js';
-import { NamespaceSchema, AddMemorySchema, ConsolidateSchema, SleepSchema, ColdTierSearchSchema, ColdTierRestoreSchema, ConfigReloadSchema, CreateDreamRunSchema, ListDreamRunsQuerySchema, AnthropicDreamCreateSchema, AnthropicPushSchema, AnthropicPullSchema, EpistemicFilterSchema, PredictSchema } from './schemas.js';
+import { NamespaceSchema, AddMemorySchema, ConsolidateSchema, SleepSchema, ColdTierSearchSchema, ColdTierRestoreSchema, ConfigReloadSchema, CreateDreamRunSchema, ListDreamRunsQuerySchema, AnthropicDreamCreateSchema, AnthropicPushSchema, AnthropicPullSchema, EpistemicFilterSchema, PredictSchema, AbstractionLevelSchema } from './schemas.js';
 import { reloadConfig } from './config.js';
 import {
   cacheGet,
@@ -659,6 +659,98 @@ export function createApp(deps: AppDependencies): express.Express {
     }
   });
 
+  // ─── Phase 5: Hierarchical Abstraction ───────────────────────────────────
+
+  /**
+   * GET /memory/:agentId/principles?[namespace=<ns>][&limit=<n>]
+   *
+   * Active principle-level abstractions, ordered by confidence then recency.
+   * The manager caps results at 50; limit (1-50) trims further.
+   */
+  app.get('/memory/:agentId/principles', requireScope('memforge:read'), async (req: Request, res: Response) => {
+    const rawNamespace = qstr(req.query['namespace']);
+    let namespace: string | undefined;
+    if (rawNamespace !== undefined) {
+      const nsResult = NamespaceSchema.safeParse(rawNamespace);
+      if (!nsResult.success) {
+        fail(res, 400, `Invalid namespace: ${nsResult.error.issues[0]?.message ?? 'validation failed'}`);
+        return;
+      }
+      namespace = nsResult.data;
+    }
+
+    const rawLimit = qstr(req.query['limit']);
+    let limit: number | undefined;
+    if (rawLimit !== undefined) {
+      limit = qnum(req.query['limit']);
+      if (limit === undefined || limit < 1 || limit > 50) {
+        fail(res, 400, '"limit" must be an integer between 1 and 50');
+        return;
+      }
+    }
+
+    let agentId: string;
+    try {
+      agentId = getAgentId(req);
+    } catch (err) {
+      fail(res, 400, (err as Error).message);
+      return;
+    }
+
+    try {
+      const principles = await manager.getPrinciples(agentId, namespace);
+      ok(res, limit !== undefined ? principles.slice(0, limit) : principles);
+    } catch (err) {
+      fail(res, 500, (err as Error).message);
+    }
+  });
+
+  /**
+   * GET /memory/:agentId/abstractions?[level=<level>][&namespace=<ns>]
+   *
+   * Active abstractions at any level, ordered by confidence then recency,
+   * capped at 50. Sleep Phase 5.11 currently writes only 'principle' rows;
+   * 'strategy' and 'mental_model' return [] until something writes them.
+   */
+  app.get('/memory/:agentId/abstractions', requireScope('memforge:read'), async (req: Request, res: Response) => {
+    const rawLevel = qstr(req.query['level']);
+    let level: AbstractionLevel | undefined;
+    if (rawLevel !== undefined) {
+      const levelResult = AbstractionLevelSchema.safeParse(rawLevel);
+      if (!levelResult.success) {
+        fail(res, 400, '"level" must be one of: principle, strategy, mental_model');
+        return;
+      }
+      level = levelResult.data;
+    }
+
+    const rawNamespace = qstr(req.query['namespace']);
+    let namespace: string | undefined;
+    if (rawNamespace !== undefined) {
+      const nsResult = NamespaceSchema.safeParse(rawNamespace);
+      if (!nsResult.success) {
+        fail(res, 400, `Invalid namespace: ${nsResult.error.issues[0]?.message ?? 'validation failed'}`);
+        return;
+      }
+      namespace = nsResult.data;
+    }
+
+    let agentId: string;
+    try {
+      agentId = getAgentId(req);
+    } catch (err) {
+      fail(res, 400, (err as Error).message);
+      return;
+    }
+
+    try {
+      const abstractions = await manager.getAbstractions(agentId, level, namespace);
+      ok(res, abstractions);
+    } catch (err) {
+      fail(res, 500, (err as Error).message);
+    }
+  });
+
   /**
    * GET /memory/:agentId/timeline?[from=<iso>][&to=<iso>][&limit=<n>]
    */

src/client.ts

@@ -60,6 +60,8 @@ import type {
   SyncStrategy,
   CausalChainNode,
   PredictionResult,
+  Abstraction,
+  AbstractionLevel,
 } from './types.js';
 
 // ─── Config ──────────────────────────────────────────────────────────────────
@@ -411,6 +413,32 @@ export class MemForgeClient {
     });
   }
 
+  // ─── Phase 5: Hierarchical Abstraction ───────────────────────────────────
+
+  /** Active principle-level abstractions (v3.11) — cross-cutting principles
+   * distilled from meta-reflections by Sleep Phase 5.11, ordered by
+   * confidence then recency. The server caps results at 50; limit (1-50)
+   * trims further. */
+  async getPrinciples(agentId: string, namespace?: string, limit?: number): Promise<Abstraction[]> {
+    const params = new URLSearchParams();
+    if (namespace) params.set('namespace', namespace);
+    if (limit !== undefined) params.set('limit', String(limit));
+    const qs = params.toString();
+    return this.get<Abstraction[]>(`/memory/${enc(agentId)}/principles${qs ? '?' + qs : ''}`);
+  }
+
+  /** Active abstractions (v3.11), optionally filtered by level, ordered by
+   * confidence then recency, capped at 50. Sleep Phase 5.11 currently writes
+   * only 'principle' rows; 'strategy' and 'mental_model' return [] until
+   * something writes them. */
+  async getAbstractions(agentId: string, level?: AbstractionLevel, namespace?: string): Promise<Abstraction[]> {
+    const params = new URLSearchParams();
+    if (level) params.set('level', level);
+    if (namespace) params.set('namespace', namespace);
+    const qs = params.toString();
+    return this.get<Abstraction[]>(`/memory/${enc(agentId)}/abstractions${qs ? '?' + qs : ''}`);
+  }
+
   /** Generate a session resumption context for an agent. */
   async resume(agentId: string, limit?: number, namespace?: string): Promise<ResumeContext> {
     const params = new URLSearchParams();
@@ -741,6 +769,14 @@ export class ResilientMemForgeClient {
     return this.safe('predict', () => this.client.predict(agentId, context, namespace), null);
   }
 
+  async getPrinciples(agentId: string, namespace?: string, limit?: number): Promise<Abstraction[]> {
+    return this.safe('getPrinciples', () => this.client.getPrinciples(agentId, namespace, limit), []);
+  }
+
+  async getAbstractions(agentId: string, level?: AbstractionLevel, namespace?: string): Promise<Abstraction[]> {
+    return this.safe('getAbstractions', () => this.client.getAbstractions(agentId, level, namespace), []);
+  }
+
   async resume(agentId: string, limit?: number, namespace?: string): Promise<ResumeContext | null> {
     return this.safe('resume', () => this.client.resume(agentId, limit, namespace), null);
   }