diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0238153 --- /dev/null +++ b/.env.example @@ -0,0 +1,52 @@ +# Composio harness configuration. +# Copy to `.env` (gitignored) and fill in. Exported process env overrides these. +# +# cp .env.example .env +# cargo run --example composio_harness --features sync + +# Required: direct-mode Composio API key. +COMPOSIO_API_KEY= + +# Optional: override the API base URL (defaults to the v3 backend). +# COMPOSIO_BASE_URL=https://backend.composio.dev/api/v3 + +# Optional: Composio user_id used to scope connected accounts. When the connect +# flow (Phase 1.5) mints a new login it uses this if set, otherwise it generates +# a `tinycortex-` id and remembers it per toolkit in the gitignored +# `.composio-harness.json` so re-runs reuse the same entity. +# COMPOSIO_ENTITY_ID= + +# Optional: comma-separated toolkits to sync (default: all connected+supported). +# Supported: gmail,github,linear,notion,clickup,slack +# Also the set the connect flow may log in when a toolkit isn't already active. +# COMPOSIO_TOOLKITS=gmail,github + +# Optional: per-toolkit ingest cap for Phase 2 (default 25). +# COMPOSIO_MAX_ITEMS=25 + +# Optional: pin/force a specific connection id per toolkit. Overrides discovery +# and can seed a toolkit that discovery missed. +# COMPOSIO_GMAIL_CONNECTION_ID= +# COMPOSIO_GITHUB_CONNECTION_ID= +# COMPOSIO_LINEAR_CONNECTION_ID= +# COMPOSIO_NOTION_CONNECTION_ID= +# COMPOSIO_CLICKUP_CONNECTION_ID= +# COMPOSIO_SLACK_CONNECTION_ID= + +# ── Login / connect flow (Phase 1.5) ───────────────────────────────────────── +# Triggered for toolkits named in COMPOSIO_TOOLKITS that have no ACTIVE account. +# +# Optional: pin the auth-config id used when initiating a login for a toolkit. +# When unset, the harness looks one up via `GET /auth_configs?toolkit_slug=...`. +# COMPOSIO_GMAIL_AUTH_CONFIG_ID= +# COMPOSIO_GITHUB_AUTH_CONFIG_ID= +# COMPOSIO_LINEAR_AUTH_CONFIG_ID= +# COMPOSIO_NOTION_AUTH_CONFIG_ID= +# COMPOSIO_CLICKUP_AUTH_CONFIG_ID= +# COMPOSIO_SLACK_AUTH_CONFIG_ID= + +# Optional: OAuth callback/redirect URL passed to the connect link. +# COMPOSIO_CALLBACK_URL= + +# Optional: how long (seconds) to poll a pending login for ACTIVE (default 120). +# COMPOSIO_CONNECT_TIMEOUT_SECS=120 diff --git a/.gitignore b/.gitignore index 762b7d0..4eedd0f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ target/ .DS_Store .DS_Store .env +.composio-harness.json .github/scripts/secrets.json .ipynb_checkpoints/ .venv diff --git a/Cargo.lock b/Cargo.lock index 4a7af3b..b270dce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -249,6 +249,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1550,6 +1556,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", + "dotenvy", "futures", "git2", "log", diff --git a/Cargo.toml b/Cargo.toml index 764944f..ec2ba10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ exclude = [ "gitbooks/", "paper/", "requirements.txt", + "viewer/", ] [features] @@ -93,6 +94,7 @@ tokio = { version = "1", features = [ ], optional = true } [dev-dependencies] +dotenvy = "0.15" tempfile = "3" tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] } wiremock = "0.6" @@ -104,3 +106,15 @@ required-features = ["sync"] [[test]] name = "composio_sync_live" required-features = ["sync"] + +# Runnable connection + memory-sync harness for a live Composio API key. +# `COMPOSIO_API_KEY=... cargo run --example composio_harness --features sync` +[[example]] +name = "composio_harness" +required-features = ["sync"] + +# Seed a workspace with sample synced memory for the viewer (no API key needed). +# `TINYCORTEX_WORKSPACE=/tmp/demo cargo run --example seed_memory --features sync` +[[example]] +name = "seed_memory" +required-features = ["sync"] diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..0145ef2 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,240 @@ +# Testing TinyCortex + +This guide covers how to exercise TinyCortex end-to-end: the Rust test suite, the +**Composio connection + memory-sync harness**, an offline **seed** for demo data, +and the **memory viewer** web app used to observe/debug what sync produced. + +- [Prerequisites](#prerequisites) +- [Rust: build & test](#rust-build--test) +- [Composio harness](#composio-harness) — connection test + live memory sync + - [Setup](#setup) + - [Environment variables](#environment-variables) + - [What each phase does](#what-each-phase-does) + - [Common runs](#common-runs) + - [Testing Gmail sync (connect flow)](#testing-gmail-sync-connect-flow) +- [Seed demo data (no API key)](#seed-demo-data-no-api-key) +- [Memory viewer](#memory-viewer) +- [End-to-end walkthrough](#end-to-end-walkthrough) +- [Troubleshooting](#troubleshooting) + +--- + +## Prerequisites + +| Tool | Version | For | +| --- | --- | --- | +| Rust | 2021 edition (stable) | the crate, harness, seed | +| Node.js | 18+ (tested on 22) | the `viewer/` web app | +| A Composio API key | direct mode (`ak_…`) | live sync only (not needed for seed) | + +The sync code is behind the `sync` cargo feature; every command below that needs +it passes `--features sync`. + +--- + +## Rust: build & test + +```sh +cargo fmt --all --check # formatting +cargo clippy --features sync # lints +cargo check # fast type-check (default features) +cargo test # unit + integration tests (default features) +cargo test --features sync # includes the Composio mock tests +``` + +Notable test targets: + +- `tests/composio_sync_mock.rs` — Composio pipelines against a `wiremock` server + (no network): pagination, cursor persistence, taint tagging, idempotency, + retry accounting. +- `tests/composio_sync_live.rs` — opt-in live checks (ignored unless configured). +- `src/memory/sync/persist_tests.rs` — the durable `KvSkillDocSink`. +- `src/memory/sync/composio/connect_tests.rs` — entity-id store + connect flow + parsing. + +--- + +## Composio harness + +`examples/composio_harness.rs` is a runnable, end-to-end check that a Composio API +key is wired correctly and that TinyCortex's sync pipelines actually ingest memory +for every connected toolkit. It doubles as a CI smoke test (non-zero exit on +failure). + +Supported toolkits (those with a TinyCortex pipeline): **gmail, github, linear, +notion, clickup, slack**. + +### Setup + +```sh +cp .env.example .env # then edit — .env is gitignored +# set at minimum: COMPOSIO_API_KEY=ak_... +``` + +The harness auto-loads `.env` from the working directory (via `dotenvy`); real +process env still overrides it. + +### Environment variables + +| Variable | Required | Purpose | +| --- | --- | --- | +| `COMPOSIO_API_KEY` | yes | Direct-mode API key. Never printed or logged. | +| `COMPOSIO_TOOLKITS` | no | Comma list restricting which toolkits to sync (e.g. `gmail,github`). Also the set the connect flow may log in. Unset = all connected+supported, non-interactive. | +| `COMPOSIO_BASE_URL` | no | Override API base (default `https://backend.composio.dev/api/v3`). | +| `COMPOSIO_ENTITY_ID` | no | Global Composio `user_id` fallback (see the user_id note below). | +| `COMPOSIO_MAX_ITEMS` | no | Per-toolkit ingest cap (default 25). | +| `TINYCORTEX_WORKSPACE` | no | When set, persist ingested documents + a `sync_manifest.json` into this workspace so the viewer can inspect them. | +| `COMPOSIO__CONNECTION_ID` | no | Pin/override a connection id, e.g. `COMPOSIO_GMAIL_CONNECTION_ID`. | +| `COMPOSIO__AUTH_CONFIG_ID` | no | Auth-config id used when initiating a login for that toolkit. Unset = looked up via `GET /auth_configs`. | +| `COMPOSIO_CALLBACK_URL` | no | OAuth callback/redirect passed to a connect link. | +| `COMPOSIO_CONNECT_TIMEOUT_SECS` | no | How long to poll a pending login for `ACTIVE` (default 120). | + +> **The `user_id` requirement (Composio v3).** Composio v3 rejects tool execution +> with HTTP 400 (`ActionExecute_ConnectedAccountEntityIdRequired`, code 1811) +> unless the account's `user_id` is sent alongside `connected_account_id`. The +> harness captures each account's `user_id` during Phase 1 discovery (and from the +> connect flow) and scopes that toolkit's transport to it, falling back to +> `COMPOSIO_ENTITY_ID`. You normally don't need to set anything — discovery +> supplies the `user_id`. + +### What each phase does + +1. **Phase 1 — connection test.** Validates the key by listing connected accounts + (`GET /connected_accounts`), printing each toolkit, its `connected_account_id`, + status, and capturing its `user_id`. +2. **Phase 1.5 — login/connect flow.** Only for toolkits named in + `COMPOSIO_TOOLKITS` that have no `ACTIVE` account. Resolves the toolkit's + auth-config, creates a Connect link scoped to a remembered per-toolkit entity id + (`.composio-harness.json`, gitignored), prints the OAuth URL, and polls until + the account is `ACTIVE` (or times out). Skipped entirely when `COMPOSIO_TOOLKITS` + is unset, so the default run stays non-interactive. +3. **Phase 2 — memory sync.** For each selected toolkit, runs the pipeline's + `tick()` twice against real Composio and grades: records ingested, provider + actions, cost, cursor advance, `taint=external_sync` tagging, and idempotency + (a second unchanged tick must ingest 0 — unless the first hit the item cap, which + is reported as `incremental`). + +### Common runs + +```sh +# All connected + supported toolkits (non-interactive): +cargo run --example composio_harness --features sync + +# Just GitHub, and persist into a workspace for the viewer: +COMPOSIO_TOOLKITS=github TINYCORTEX_WORKSPACE=/tmp/tinycortex-live \ + cargo run --example composio_harness --features sync +``` + +A healthy run ends with `HARNESS PASS` and a table like: + +``` +toolkit result recs acts cost$ taint idempotency notes +github PASS 50 2 0.0000 ok incremental conn=ca_… cursor=none +``` + +### Testing Gmail sync (connect flow) + +If no Gmail account is connected yet, request it explicitly and the harness will +run the **Phase 1.5 login flow**: + +```sh +COMPOSIO_TOOLKITS=gmail TINYCORTEX_WORKSPACE=/tmp/tinycortex-live \ + cargo run --example composio_harness --features sync +``` + +It prints an OAuth URL — open it, authorize Gmail, and the harness polls until the +connection is `ACTIVE`, then syncs it. This needs a Gmail auth-config in your +Composio dashboard; pin one explicitly with `COMPOSIO_GMAIL_AUTH_CONFIG_ID=…` if +auto-lookup can't find it. Once connected, subsequent runs reuse the account (the +entity id is remembered in `.composio-harness.json`). + +--- + +## Seed demo data (no API key) + +`examples/seed_memory.rs` builds a realistic workspace with **no Composio key**: it +persists sample skill documents (via the same `KvSkillDocSink` sync uses) and builds +a real summary tree (chunks → L0 → L1/L2 via the offline `ConcatSummariser`). Use it +to exercise the viewer — including the memory-graph view — offline. + +```sh +TINYCORTEX_WORKSPACE=/tmp/tinycortex-demo \ + cargo run --example seed_memory --features sync +``` + +--- + +## Memory viewer + +A local, read-only Next.js app (`viewer/`) whose **server-side** code opens the +workspace's SQLite stores read-only and reads the sync artifacts. No network; the +DB is opened read-only. + +```sh +cd viewer +npm install +TINYCORTEX_WORKSPACE=/tmp/tinycortex-demo npm run dev +# → http://localhost:4319 +``` + +Checks: + +```sh +npx tsc --noEmit # type-check +npm run build # production build +``` + +Views: + +| Route | Shows | +| --- | --- | +| `/` | Overview — which stores exist, doc/toolkit/chunk/entity counts | +| `/docs` | Skill documents ingested by sync, filter by toolkit + search, detail with content/metadata/raw payload | +| `/graph` | Force-directed summary tree (root → source → summary levels → chunks); pan/zoom/drag/hover/fit | +| `/tree` | Memory-tree chunks (`memory_tree/chunks.db`) | +| `/entities` | Canonical entities under `memory_tree/content/entities/` | +| `/runs` | The last sync run's per-toolkit results + event stream (`sync_manifest.json`) | + +The viewer reads the workspace on every request — re-run a sync and refresh to see +new state. It must point at the **same** `TINYCORTEX_WORKSPACE` the harness/seed +wrote to. + +> Note: the harness persists **skill documents** into a workspace; it does not build +> the full chunk/summary tree (that is a separate ingest pass). So the `/graph` and +> `/tree` views are populated by the **seed** example (or a full ingest), while +> `/docs` and `/runs` reflect live harness sync output. + +--- + +## End-to-end walkthrough + +```sh +# 1. Sync real data into a workspace (GitHub shown; see Gmail section for OAuth): +COMPOSIO_TOOLKITS=github TINYCORTEX_WORKSPACE=/tmp/tinycortex-live \ + cargo run --example composio_harness --features sync +# → HARNESS PASS, N documents persisted to /tmp/tinycortex-live + +# 2. Observe it: +cd viewer && npm install +TINYCORTEX_WORKSPACE=/tmp/tinycortex-live npm run dev +# → open http://localhost:4319/docs (and /runs for the run manifest) + +# 3. For an offline demo with a full memory graph, seed a second workspace: +TINYCORTEX_WORKSPACE=/tmp/tinycortex-demo \ + cargo run --example seed_memory --features sync +# restart the viewer against /tmp/tinycortex-demo and open /graph +``` + +--- + +## Troubleshooting + +| Symptom | Cause / fix | +| --- | --- | +| `HTTP 400 … User ID is required` | Old build. Update — the harness now sends each account's `user_id`. If a connection has none, set `COMPOSIO_ENTITY_ID`. | +| `connected_accounts returned HTTP 401` | Invalid/expired `COMPOSIO_API_KEY`. | +| `No supported+connected toolkits to sync` | None of your connected accounts map to a supported toolkit, or all are `EXPIRED`. Connect one (or use the Gmail connect flow), or set `COMPOSIO__CONNECTION_ID`. | +| Toolkit account shows `EXPIRED` | Re-authorize it; the harness treats terminal-status accounts as absent so the connect flow can re-establish them. | +| `/graph` or `/tree` empty in the viewer | That workspace has skill docs but no memory tree. Use the seed example or point at a workspace produced by a full ingest. | +| Viewer shows "No workspace configured" | `TINYCORTEX_WORKSPACE` is unset for the `npm run dev` process. | +| Secrets in synced content | Expected to be redacted: KV writes sanitize values, so the viewer never surfaces raw credentials. | diff --git a/examples/composio_harness.rs b/examples/composio_harness.rs new file mode 100644 index 0000000..4728d47 --- /dev/null +++ b/examples/composio_harness.rs @@ -0,0 +1,799 @@ +//! Composio connection + memory-sync harness. +//! +//! An end-to-end, runnable check that a Composio API key is wired correctly and +//! that TinyCortex's Composio sync pipelines actually ingest memory for every +//! connected toolkit. +//! +//! ## Usage +//! +//! Either export the key inline, or copy `.env.example` to `.env` and fill it +//! in (`.env` is gitignored; the harness loads it automatically, and real +//! process env still overrides it): +//! +//! ```sh +//! cp .env.example .env # then edit +//! cargo run --example composio_harness --features sync +//! # or: +//! COMPOSIO_API_KEY=ak_... cargo run --example composio_harness --features sync +//! ``` +//! +//! ### Phase 1 — Connection test +//! Validates the API key by listing connected accounts against the Composio v3 +//! API (`GET /connected_accounts`). This proves the key is live and discovers +//! which toolkits are connected together with their `connected_account_id`s. +//! +//! ### Phase 1.5 — Login / connect flow +//! When you request specific toolkits with `COMPOSIO_TOOLKITS` and one of them +//! has no ACTIVE connected account (discovery found none, or its status is a +//! terminal FAILED/EXPIRED/REVOKED), the harness initiates a Composio Connect +//! link for it: it resolves the toolkit's auth-config id (or an explicit +//! `COMPOSIO__AUTH_CONFIG_ID`), creates a connection scoped to a +//! remembered per-toolkit entity id, prints the OAuth URL for you to open, then +//! polls until the account is ACTIVE (or times out) before syncing it. This +//! step is skipped entirely when `COMPOSIO_TOOLKITS` is unset, so the default +//! run stays non-interactive and CI-safe. +//! +//! ### Phase 2 — Memory sync +//! For every discovered toolkit that TinyCortex has a pipeline for (Gmail, +//! GitHub, Linear, Notion, ClickUp, Slack), the harness runs `tick()` twice +//! against real Composio using in-memory sinks, then reports records ingested, +//! provider actions, cost, cursor advance, taint tagging, and idempotency. +//! +//! The process exits non-zero if the connection test fails or if any toolkit's +//! sync errors, so the harness is usable as a CI smoke test. +//! +//! ## Environment +//! - `COMPOSIO_API_KEY` (required) direct-mode API key. +//! - `COMPOSIO_BASE_URL` override the API base (default v3 backend). +//! - `COMPOSIO_ENTITY_ID` Composio `user_id` scoping accounts. When the +//! connect flow mints a new connection it uses this if set, else a generated +//! `tinycortex-` remembered per toolkit in `.composio-harness.json`. +//! - `COMPOSIO_TOOLKITS` comma list restricting which toolkits to sync; +//! also the set the connect flow may log in when not already active. +//! - `COMPOSIO_MAX_ITEMS` per-toolkit ingest cap (default 25). +//! - `COMPOSIO__CONNECTION_ID` pin/override a connection id, e.g. +//! `COMPOSIO_GMAIL_CONNECTION_ID`. Also seeds a toolkit if discovery is empty. +//! - `COMPOSIO__AUTH_CONFIG_ID` pin the auth-config id used when +//! initiating a login for that toolkit, e.g. `COMPOSIO_GMAIL_AUTH_CONFIG_ID`. +//! When unset the harness looks one up via `GET /auth_configs`. +//! - `COMPOSIO_CALLBACK_URL` optional OAuth callback/redirect passed to the +//! connect link. +//! - `COMPOSIO_CONNECT_TIMEOUT_SECS` how long to poll a pending login for +//! `ACTIVE` before failing (default 120). +//! - `TINYCORTEX_WORKSPACE` when set, persist ingested documents (via +//! `KvSkillDocSink`) and a `sync_manifest.json` run manifest into this +//! workspace directory so the memory viewer can inspect what sync produced. + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::Mutex; + +use async_trait::async_trait; +use serde_json::Value; +use tinycortex::memory::config::{ComposioMode, ComposioSyncConfig, MemoryConfig, SecretString}; +use tinycortex::memory::sync::{ + create_connection_link, get_connection_status, list_auth_configs, resolve_auth_config_id, + status_is_active, status_is_terminal, ClickUpSyncPipeline, ComposioClient, EntityStore, + GitHubSyncPipeline, GmailSyncPipeline, KvSkillDocSink, LinearSyncPipeline, NotionSyncPipeline, + SkillDocSink, SkillDocument, SlackSyncPipeline, SyncContext, SyncEvent, SyncEventSink, + SyncPipeline, SyncState, SyncStateStore, +}; + +const DEFAULT_BASE_URL: &str = "https://backend.composio.dev/api/v3"; +const DEFAULT_MAX_ITEMS: u32 = 25; +/// Local, gitignored file remembering the entity id (`user_id`) per toolkit. +const ENTITY_STORE_FILE: &str = ".composio-harness.json"; +/// How long to poll a pending connection for `ACTIVE` before giving up. +const DEFAULT_CONNECT_TIMEOUT_SECS: u64 = 120; +/// Seconds between connection-status polls. +const CONNECT_POLL_INTERVAL_SECS: u64 = 3; + +/// Toolkits with a TinyCortex sync pipeline, in a stable report order. +const SUPPORTED_TOOLKITS: &[&str] = &["gmail", "github", "linear", "notion", "clickup", "slack"]; + +// ── In-memory host sinks ──────────────────────────────────────────────────── +// The harness stands in for a real host: it captures skill documents, sync +// events, and cursor/dedup state entirely in process so nothing is persisted. + +#[derive(Default)] +struct Captures { + documents: Mutex>, + events: Mutex>, + state: Mutex>, +} + +#[async_trait] +impl SkillDocSink for Captures { + async fn store(&self, document: SkillDocument) -> anyhow::Result<()> { + self.documents.lock().unwrap().push(document); + Ok(()) + } + + async fn delete(&self, namespace_skill_id: &str, document_id: &str) -> anyhow::Result<()> { + self.documents.lock().unwrap().retain(|document| { + document.namespace_skill_id != namespace_skill_id || document.document_id != document_id + }); + Ok(()) + } +} + +#[async_trait] +impl SyncEventSink for Captures { + async fn emit(&self, event: SyncEvent) -> anyhow::Result<()> { + self.events.lock().unwrap().push(event); + Ok(()) + } +} + +#[async_trait] +impl SyncStateStore for Captures { + async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { + Ok(self + .state + .lock() + .unwrap() + .get(&format!("{namespace}:{key}")) + .cloned()) + } + + async fn set(&self, namespace: &str, key: &str, value: &Value) -> anyhow::Result<()> { + self.state + .lock() + .unwrap() + .insert(format!("{namespace}:{key}"), value.clone()); + Ok(()) + } +} + +impl Captures { + fn context(self: &Arc) -> SyncContext { + SyncContext { + events: self.clone(), + documents: self.clone(), + state: self.clone(), + local_documents: None, + external_sources: None, + summariser: None, + } + } + + fn docs_for(&self, toolkit: &str) -> Vec { + self.documents + .lock() + .unwrap() + .iter() + .filter(|doc| doc.toolkit == toolkit) + .cloned() + .collect() + } +} + +// ── A connected account discovered from the Composio API ──────────────────── + +struct Connection { + toolkit: String, + connection_id: String, + status: Option, + /// The account's Composio `user_id`. Composio v3 requires it alongside + /// `connected_account_id` on every tool execution, so it is captured here + /// during discovery and passed through to the sync transport. + user_id: Option, +} + +fn env_opt(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) +} + +/// Best-effort extraction of a toolkit slug from a connected-account record. +/// Composio has shipped several shapes over v3; probe the common ones. +fn record_toolkit(record: &Value) -> Option { + let candidates = [ + record.pointer("/toolkit/slug"), + record.pointer("/toolkit/name"), + record.get("toolkit"), + record.get("appName"), + record.get("appUniqueId"), + record.pointer("/app/uniqueKey"), + record.pointer("/app/name"), + ]; + candidates + .into_iter() + .flatten() + .find_map(Value::as_str) + .map(|slug| slug.trim().to_ascii_lowercase()) + .filter(|slug| !slug.is_empty()) +} + +fn record_id(record: &Value) -> Option { + ["id", "connectedAccountId", "connected_account_id", "nanoid"] + .iter() + .find_map(|key| record.get(key).and_then(Value::as_str)) + .map(str::trim) + .filter(|id| !id.is_empty()) + .map(str::to_owned) +} + +/// Phase 1: validate the key and list connected accounts. +async fn discover_connections( + http: &reqwest::Client, + base_url: &str, + api_key: &str, + entity_id: Option<&str>, +) -> anyhow::Result> { + let mut request = http + .get(format!( + "{}/connected_accounts", + base_url.trim_end_matches('/') + )) + .header("x-api-key", api_key); + if let Some(entity_id) = entity_id { + request = request.query(&[("user_ids", entity_id)]); + } + let response = request + .send() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts request failed: {error}"))?; + let status = response.status(); + if !status.is_success() { + // Never echo the response body; it can contain the key back verbatim. + let _ = response.bytes().await; + anyhow::bail!("connected_accounts returned HTTP {status}"); + } + let body: Value = response + .json() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts decode failed: {error}"))?; + + let items = ["/items", "/data/items", "/data", "/connectedAccounts"] + .iter() + .find_map(|pointer| body.pointer(pointer).and_then(Value::as_array)) + .or_else(|| body.as_array()) + .cloned() + .unwrap_or_default(); + + let mut connections = Vec::new(); + for record in &items { + let (Some(toolkit), Some(connection_id)) = (record_toolkit(record), record_id(record)) + else { + continue; + }; + connections.push(Connection { + toolkit, + connection_id, + status: record + .get("status") + .and_then(Value::as_str) + .map(str::to_owned), + user_id: record + .get("user_id") + .and_then(Value::as_str) + .map(str::trim) + .filter(|id| !id.is_empty()) + .map(str::to_owned), + }); + } + Ok(connections) +} + +/// Phase 1.5: drive a Composio Connect login for one toolkit and return an +/// ACTIVE connection, or an error carrying manual next steps. +/// +/// Resolves an auth-config id (explicit override or `GET /auth_configs`), +/// creates a connect link scoped to `entity_id`, prints the OAuth URL, then +/// polls `GET /connected_accounts/{id}` until the account is ACTIVE, hits a +/// terminal failure, or `timeout` elapses. Never prints secrets. +#[allow(clippy::too_many_arguments)] +async fn connect_toolkit( + http: &reqwest::Client, + base_url: &str, + api_key: &str, + toolkit: &str, + entity_id: &str, + auth_config_override: Option<&str>, + callback_url: Option<&str>, + timeout: std::time::Duration, +) -> anyhow::Result { + // 1. Resolve the auth-config id for this toolkit. + let auth_config_id = match auth_config_override { + Some(id) => id.to_owned(), + None => { + let list = list_auth_configs(http, base_url, api_key, Some(toolkit)).await?; + resolve_auth_config_id(&list, toolkit).ok_or_else(|| { + anyhow::anyhow!( + "no auth config found for '{toolkit}'. Create one in the Composio dashboard \ + or set COMPOSIO_{}_AUTH_CONFIG_ID.", + toolkit.to_ascii_uppercase() + ) + })? + } + }; + println!(" auth config: {auth_config_id}"); + + // 2. Create the Composio Connect link. + let link = create_connection_link( + http, + base_url, + api_key, + &auth_config_id, + entity_id, + callback_url, + ) + .await?; + + // 3. Prompt the user to complete the browser login. + match &link.redirect_url { + Some(url) => { + println!(" action required — open this URL to authorize {toolkit}, then return here:"); + println!(" {url}"); + } + None => println!(" no browser step reported for {toolkit}; waiting for activation…"), + } + println!( + " polling connection {} for up to {}s…", + link.connected_account_id, + timeout.as_secs() + ); + + // 4. Poll until ACTIVE / terminal / timeout. + let deadline = std::time::Instant::now() + timeout; + loop { + match get_connection_status(http, base_url, api_key, &link.connected_account_id).await { + Ok(Some(status)) if status_is_active(&status) => { + println!(" {toolkit} connection is ACTIVE."); + return Ok(Connection { + toolkit: toolkit.to_owned(), + connection_id: link.connected_account_id, + status: Some(status), + // The connection was minted for exactly this entity id. + user_id: Some(entity_id.to_owned()), + }); + } + Ok(Some(status)) if status_is_terminal(&status) => { + anyhow::bail!("{toolkit} login ended in terminal state {status}; re-run to retry.") + } + Ok(Some(status)) => println!(" status: {status} …"), + Ok(None) => println!(" status: (unknown) …"), + // A transient poll error shouldn't abort the whole wait. + Err(error) => println!(" poll error: {error}"), + } + if std::time::Instant::now() >= deadline { + anyhow::bail!( + "timed out waiting for {toolkit} to become ACTIVE after {}s. Finish the login in \ + the browser, then re-run — the entity id is remembered in {}.", + timeout.as_secs(), + ENTITY_STORE_FILE + ); + } + tokio::time::sleep(std::time::Duration::from_secs(CONNECT_POLL_INTERVAL_SECS)).await; + } +} + +fn build_pipeline( + toolkit: &str, + client: ComposioClient, + connection_id: &str, +) -> Option> { + match toolkit { + "gmail" => Some(Box::new(GmailSyncPipeline::new(client, connection_id))), + "github" => Some(Box::new(GitHubSyncPipeline::new(client, connection_id))), + "linear" => Some(Box::new(LinearSyncPipeline::new(client, connection_id))), + "notion" => Some(Box::new(NotionSyncPipeline::new(client, connection_id))), + "clickup" => Some(Box::new(ClickUpSyncPipeline::new(client, connection_id))), + "slack" => Some(Box::new(SlackSyncPipeline::new(client, connection_id))), + _ => None, + } +} + +struct ToolkitReport { + toolkit: String, + connection_id: String, + ingested: u32, + actions: u32, + cost_usd: f64, + docs_stored: usize, + taint_ok: bool, + cursor_advanced: bool, + idempotency: &'static str, + error: Option, +} + +impl ToolkitReport { + fn passed(&self) -> bool { + self.error.is_none() && self.idempotency != "FAIL" + } +} + +/// Phase 2: run one toolkit's pipeline twice and grade the outcome. +async fn run_toolkit( + toolkit: &str, + connection_id: &str, + config: &MemoryConfig, + transport: &ComposioSyncConfig, + captures: &Arc, +) -> ToolkitReport { + let mut report = ToolkitReport { + toolkit: toolkit.to_owned(), + connection_id: connection_id.to_owned(), + ingested: 0, + actions: 0, + cost_usd: 0.0, + docs_stored: 0, + taint_ok: true, + cursor_advanced: false, + idempotency: "n/a", + error: None, + }; + + let client = ComposioClient::new(transport.clone()); + let Some(pipeline) = build_pipeline(toolkit, client, connection_id) else { + report.error = Some("no TinyCortex pipeline for toolkit".into()); + return report; + }; + let context = captures.context(); + + let first = match pipeline.tick(config, &context).await { + Ok(outcome) => outcome, + Err(error) => { + report.error = Some(error.to_string()); + return report; + } + }; + report.ingested = first.records_ingested; + report.actions = first.actions_called; + report.cost_usd = first.provider_cost_usd; + + let docs = captures.docs_for(toolkit); + report.docs_stored = docs.len(); + report.taint_ok = docs + .iter() + .all(|doc| doc.metadata.get("taint").and_then(Value::as_str) == Some("external_sync")); + + if let Ok(state) = SyncState::load(captures.as_ref(), toolkit, connection_id).await { + report.cursor_advanced = state.cursor.is_some(); + } + + // A second tick against unchanged upstream data must not re-ingest anything + // — unless the first run was capped (`more_pending`), in which case picking + // up further items is correct incremental behaviour, not a dedup failure. + match pipeline.tick(config, &context).await { + Ok(second) if first.more_pending => { + report.idempotency = "incremental"; + report.ingested = report.ingested.saturating_add(second.records_ingested); + } + Ok(second) if second.records_ingested == 0 => report.idempotency = "PASS", + Ok(second) => { + report.idempotency = "FAIL"; + report.error = Some(format!( + "second tick re-ingested {} records", + second.records_ingested + )); + } + Err(error) => { + report.idempotency = "FAIL"; + report.error = Some(format!("second tick errored: {error}")); + } + } + report +} + +fn print_report(reports: &[ToolkitReport]) { + println!("\n── Sync results ──────────────────────────────────────────────"); + println!( + "{:<9} {:<7} {:>5} {:>5} {:>8} {:<6} {:<12} notes", + "toolkit", "result", "recs", "acts", "cost$", "taint", "idempotency" + ); + for report in reports { + println!( + "{:<9} {:<7} {:>5} {:>5} {:>8.4} {:<6} {:<12} {}", + report.toolkit, + if report.passed() { "PASS" } else { "FAIL" }, + report.ingested, + report.actions, + report.cost_usd, + if report.taint_ok { "ok" } else { "MISS" }, + report.idempotency, + report + .error + .as_deref() + .map(|error| format!("error: {error}")) + .unwrap_or_else(|| format!( + "conn={} cursor={}", + report.connection_id, + if report.cursor_advanced { + "advanced" + } else { + "none" + } + )), + ); + } +} + +/// Persist captured skill documents into `workspace` via the durable +/// [`KvSkillDocSink`], and drop a JSON run manifest (per-toolkit results plus +/// the full sync-event stream) at `/sync_manifest.json`. This is +/// what the memory viewer reads to show what a sync run ingested. +async fn persist_to_workspace( + workspace: &str, + captures: &Arc, + reports: &[ToolkitReport], +) -> anyhow::Result<()> { + let root = std::path::Path::new(workspace); + let sink = KvSkillDocSink::open_in_workspace(root)?; + let docs = captures.documents.lock().unwrap().clone(); + let stored = docs.len(); + for doc in docs { + sink.store(doc).await?; + } + + let toolkits: Vec = reports + .iter() + .map(|report| { + serde_json::json!({ + "toolkit": report.toolkit, + "connectionId": report.connection_id, + "ingested": report.ingested, + "actions": report.actions, + "costUsd": report.cost_usd, + "docsStored": report.docs_stored, + "taintOk": report.taint_ok, + "cursorAdvanced": report.cursor_advanced, + "idempotency": report.idempotency, + "passed": report.passed(), + "error": report.error, + }) + }) + .collect(); + let events = serde_json::to_value(&*captures.events.lock().unwrap())?; + let manifest = serde_json::json!({ + "toolkits": toolkits, + "events": events, + "documentsPersisted": stored, + }); + let manifest_path = root.join("sync_manifest.json"); + if let Some(parent) = manifest_path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; + + println!( + "\nPersisted {stored} document(s) to {} and a run manifest to {}.", + root.join(tinycortex::memory::sync::SKILL_DOCS_DB).display(), + manifest_path.display() + ); + Ok(()) +} + +#[tokio::main] +async fn main() { + if let Err(error) = run().await { + eprintln!("\nharness aborted: {error}"); + std::process::exit(1); + } +} + +async fn run() -> anyhow::Result<()> { + // Load a `.env` from the working dir (and any ancestor) if present. Existing + // process env always wins, so `COMPOSIO_API_KEY=... cargo run` still works. + match dotenvy::dotenv() { + Ok(path) => println!("loaded env from {}", path.display()), + Err(error) if error.not_found() => {} + Err(error) => eprintln!("warning: could not read .env: {error}"), + } + + let api_key = env_opt("COMPOSIO_API_KEY") + .ok_or_else(|| anyhow::anyhow!("COMPOSIO_API_KEY is required"))?; + let base_url = env_opt("COMPOSIO_BASE_URL").unwrap_or_else(|| DEFAULT_BASE_URL.to_owned()); + let entity_id = env_opt("COMPOSIO_ENTITY_ID"); + let max_items = env_opt("COMPOSIO_MAX_ITEMS") + .and_then(|value| value.parse::().ok()) + .unwrap_or(DEFAULT_MAX_ITEMS); + let toolkit_filter: Option> = env_opt("COMPOSIO_TOOLKITS").map(|list| { + list.split(',') + .map(|item| item.trim().to_ascii_lowercase()) + .filter(|item| !item.is_empty()) + .collect() + }); + // When set, persist ingested documents + a run manifest into this workspace + // so the memory viewer (or any host) can inspect what sync produced. + let workspace = env_opt("TINYCORTEX_WORKSPACE"); + + let http = reqwest::Client::new(); + + // ── Phase 1: connection test ──────────────────────────────────────────── + println!("── Phase 1: connection test ──────────────────────────────────"); + println!("base_url : {base_url}"); + println!("entity : {}", entity_id.as_deref().unwrap_or("(none)")); + let mut connections = + discover_connections(&http, &base_url, &api_key, entity_id.as_deref()).await?; + println!( + "connection OK — {} connected account(s) discovered", + connections.len() + ); + for connection in &connections { + println!( + " • {:<9} {} [{}]", + connection.toolkit, + connection.connection_id, + connection.status.as_deref().unwrap_or("unknown") + ); + } + + // Explicit `COMPOSIO__CONNECTION_ID` overrides win and can seed a + // toolkit that discovery missed (e.g. a freshly connected account). + for toolkit in SUPPORTED_TOOLKITS { + if let Some(connection_id) = env_opt(&format!( + "COMPOSIO_{}_CONNECTION_ID", + toolkit.to_ascii_uppercase() + )) { + connections.retain(|connection| connection.toolkit != *toolkit); + connections.push(Connection { + toolkit: (*toolkit).to_owned(), + connection_id, + status: Some("env-override".into()), + // Unknown for a pinned id; Phase 2 falls back to COMPOSIO_ENTITY_ID. + user_id: None, + }); + } + } + + // Keep only toolkits we can actually sync, honouring an optional filter, and + // collapse to one connection per toolkit (the first discovered). A + // connection whose status is a terminal failure is treated as absent so the + // connect flow can re-establish it. + let mut selected: Vec = Vec::new(); + for connection in connections { + if !SUPPORTED_TOOLKITS.contains(&connection.toolkit.as_str()) { + continue; + } + if toolkit_filter + .as_ref() + .is_some_and(|filter| !filter.contains(&connection.toolkit)) + { + continue; + } + if connection.status.as_deref().is_some_and(status_is_terminal) { + continue; + } + if selected + .iter() + .any(|kept| kept.toolkit == connection.toolkit) + { + continue; + } + selected.push(connection); + } + + // ── Phase 1.5: login/connect flow for requested-but-missing toolkits ──── + // Only runs for explicitly requested toolkits so the default (no filter) + // run stays non-interactive. Each new login reuses/records a per-toolkit + // entity id so re-runs don't orphan connections. + if let Some(requested) = toolkit_filter.as_ref() { + let store_path = std::env::current_dir() + .unwrap_or_default() + .join(ENTITY_STORE_FILE); + let mut entity_store = EntityStore::load(&store_path); + let connect_timeout = std::time::Duration::from_secs( + env_opt("COMPOSIO_CONNECT_TIMEOUT_SECS") + .and_then(|value| value.parse::().ok()) + .unwrap_or(DEFAULT_CONNECT_TIMEOUT_SECS), + ); + let callback_url = env_opt("COMPOSIO_CALLBACK_URL"); + + for toolkit in requested { + if !SUPPORTED_TOOLKITS.contains(&toolkit.as_str()) { + continue; + } + if selected.iter().any(|kept| kept.toolkit == *toolkit) { + continue; + } + println!("\n── Phase 1.5: connecting {toolkit} ───────────────────────────"); + let toolkit_entity = match entity_store.entity_id_for(toolkit, entity_id.as_deref()) { + Ok(id) => id, + Err(error) => { + println!(" could not persist entity id for {toolkit}: {error}"); + continue; + } + }; + println!(" entity id: {toolkit_entity} (stored in {ENTITY_STORE_FILE})"); + let auth_config_override = env_opt(&format!( + "COMPOSIO_{}_AUTH_CONFIG_ID", + toolkit.to_ascii_uppercase() + )); + match connect_toolkit( + &http, + &base_url, + &api_key, + toolkit, + &toolkit_entity, + auth_config_override.as_deref(), + callback_url.as_deref(), + connect_timeout, + ) + .await + { + Ok(connection) => selected.push(connection), + Err(error) => println!(" connect failed for {toolkit}: {error}"), + } + } + } + + if selected.is_empty() { + println!("\nNo supported+connected toolkits to sync. Connect an account in Composio"); + println!("or set COMPOSIO__CONNECTION_ID to force one. Phase 1 passed."); + return Ok(()); + } + + // ── Phase 2: memory sync ──────────────────────────────────────────────── + println!("\n── Phase 2: memory sync ({max_items} item cap/toolkit) ───────"); + let api_secret = SecretString::new(api_key); + let tmp = std::env::temp_dir().join("tinycortex-composio-harness"); + let mut config = MemoryConfig::new(tmp.to_string_lossy().into_owned()); + config.sync.budget.max_items = Some(max_items); + + let captures = Arc::new(Captures::default()); + let mut reports = Vec::new(); + for connection in &selected { + // Composio v3 requires the account's user_id alongside its + // connected_account_id, so scope each toolkit's transport to that + // account's user_id (falling back to a global COMPOSIO_ENTITY_ID). + let conn_entity = connection.user_id.clone().or_else(|| entity_id.clone()); + if conn_entity.is_none() { + println!( + " ! no user_id for {} — set COMPOSIO_ENTITY_ID; execution will 400", + connection.toolkit + ); + } + let transport = ComposioSyncConfig { + mode: ComposioMode::Direct, + base_url: base_url.clone(), + api_key: Some(api_secret.clone()), + bearer_token: None, + entity_id: conn_entity.clone(), + }; + println!( + "→ syncing {} ({}, user_id={})", + connection.toolkit, + connection.connection_id, + conn_entity.as_deref().unwrap_or("(none)") + ); + let report = run_toolkit( + &connection.toolkit, + &connection.connection_id, + &config, + &transport, + &captures, + ) + .await; + reports.push(report); + } + + print_report(&reports); + + let total_docs = captures.documents.lock().unwrap().len(); + let passed = reports.iter().filter(|report| report.passed()).count(); + println!( + "\nSummary: {passed}/{} toolkit(s) passed, {total_docs} document(s) ingested into memory.", + reports.len() + ); + + if let Some(workspace) = workspace.as_deref() { + if let Err(error) = persist_to_workspace(workspace, &captures, &reports).await { + eprintln!("warning: could not persist to workspace {workspace}: {error}"); + } + } + + if passed == reports.len() { + println!("HARNESS PASS"); + Ok(()) + } else { + anyhow::bail!( + "{}/{} toolkit(s) failed", + reports.len() - passed, + reports.len() + ) + } +} diff --git a/examples/seed_memory.rs b/examples/seed_memory.rs new file mode 100644 index 0000000..6ad406d --- /dev/null +++ b/examples/seed_memory.rs @@ -0,0 +1,328 @@ +//! Seed a workspace with sample synced memory for the viewer. +//! +//! Writes: +//! 1. a handful of [`SkillDocument`]s through the durable [`KvSkillDocSink`] +//! plus a `sync_manifest.json` (the "skill docs" + "sync runs" views), and +//! 2. a real summary **memory tree** (chunks → L0 → L1 summaries) built with +//! the offline [`ConcatSummariser`], so the graph/tree views have data. +//! +//! No live Composio API key is required. +//! +//! ```sh +//! TINYCORTEX_WORKSPACE=/tmp/tinycortex-demo \ +//! cargo run --example seed_memory --features sync +//! ``` +//! +//! If `TINYCORTEX_WORKSPACE` is unset it defaults to +//! `/tinycortex-memory-demo`. + +use chrono::{TimeZone, Utc}; +use serde_json::json; + +use tinycortex::memory::chunks::{chunk_id, upsert_chunks, Chunk, Metadata, SourceKind, SourceRef}; +use tinycortex::memory::config::MemoryConfig; +use tinycortex::memory::sync::{KvSkillDocSink, SkillDocSink, SkillDocument, SKILL_DOCS_DB}; +use tinycortex::memory::tree::{ConcatSummariser, LeafRef, TreeFactory}; + +// ── Skill documents (the "skill docs" + "sync runs" views) ────────────────── + +fn doc(toolkit: &str, id: &str, title: &str, content: &str) -> SkillDocument { + SkillDocument { + namespace_skill_id: toolkit.into(), + connection_id: format!("ca_demo_{toolkit}"), + document_id: format!("{toolkit}:{id}"), + title: title.into(), + content: content.into(), + toolkit: toolkit.into(), + metadata: json!({ + "source": "composio-provider-incremental", + "taint": "external_sync", + "provider_id": id, + }), + } +} + +async fn seed_skill_docs(root: &std::path::Path) -> anyhow::Result { + let docs = [ + doc( + "gmail", + "18f2a1", + "Re: Q3 planning sync", + "Thanks for the notes. Let's lock the roadmap review for Thursday and \ + pull the metrics dashboard into the deck beforehand.", + ), + doc( + "gmail", + "18f2b7", + "Invoice #4021 from Acme Cloud", + "Your monthly invoice is attached. Amount due: $412.00. \ + Auto-pay will run on the 5th.", + ), + doc( + "github", + "pr-882", + "harden queue and tree concurrency (#882)", + "Adds a busy-timeout to the chunk pool and serializes tree seals. \ + Fixes intermittent SQLITE_BUSY under parallel ingest.", + ), + doc( + "github", + "issue-66", + "port OpenHuman engine gaps", + "Tracking parity work: entity index, summary fan-out, and the \ + periodic sync cadence.", + ), + doc( + "linear", + "TIN-140", + "Build memory debug viewer", + "Next.js app that inspects the local workspace: skill docs, the \ + memory tree, and sync run manifests.", + ), + ]; + + let sink = KvSkillDocSink::open_in_workspace(root)?; + for document in docs.iter().cloned() { + sink.store(document).await?; + } + + let manifest = json!({ + "toolkits": [ + { "toolkit": "gmail", "connectionId": "ca_demo_gmail", "ingested": 2, + "actions": 2, "costUsd": 0.0, "docsStored": 2, "taintOk": true, + "cursorAdvanced": true, "idempotency": "PASS", "passed": true, "error": null }, + { "toolkit": "github", "connectionId": "ca_demo_github", "ingested": 2, + "actions": 3, "costUsd": 0.0, "docsStored": 2, "taintOk": true, + "cursorAdvanced": true, "idempotency": "PASS", "passed": true, "error": null }, + { "toolkit": "linear", "connectionId": "ca_demo_linear", "ingested": 1, + "actions": 1, "costUsd": 0.0, "docsStored": 1, "taintOk": true, + "cursorAdvanced": true, "idempotency": "PASS", "passed": true, "error": null } + ], + "events": [ + { "sourceId": "gmail", "toolkit": "gmail", "connectionId": "ca_demo_gmail", "stage": "completed" }, + { "sourceId": "github", "toolkit": "github", "connectionId": "ca_demo_github", "stage": "completed" }, + { "sourceId": "linear", "toolkit": "linear", "connectionId": "ca_demo_linear", "stage": "completed" } + ], + "documentsPersisted": docs.len(), + }); + std::fs::create_dir_all(root)?; + std::fs::write( + root.join("sync_manifest.json"), + serde_json::to_vec_pretty(&manifest)?, + )?; + Ok(docs.len()) +} + +// ── Memory tree (the "graph" + "memory tree" views) ───────────────────────── + +/// One leaf message: (content, entities, topics). +type Message = ( + &'static str, + &'static [&'static str], + &'static [&'static str], +); + +/// Seed one source tree: persist a chunk per message, insert each as a leaf, +/// then force-seal. A high per-leaf token count seals every leaf into its own +/// L0 summary, and once `SUMMARY_FANOUT` L0s accumulate an L1 seals — producing +/// a visible chunk → L0 → L1 hierarchy. +async fn seed_tree( + cfg: &MemoryConfig, + kind: SourceKind, + scope: &str, + owner: &str, + messages: &[Message], +) -> anyhow::Result<()> { + let factory = TreeFactory::source(scope); + let summariser = ConcatSummariser::new(); + let base_ms = 1_700_000_000_000_i64; + + for (seq, (content, entities, topics)) in messages.iter().enumerate() { + let ts = Utc + .timestamp_millis_opt(base_ms + seq as i64 * 60_000) + .single() + .expect("valid timestamp"); + let chunk = Chunk { + id: chunk_id(kind, scope, seq as u32, content), + content: (*content).to_string(), + metadata: Metadata { + source_kind: kind, + source_id: scope.to_string(), + owner: owner.to_string(), + timestamp: ts, + time_range: (ts, ts), + tags: vec![scope.split(':').next().unwrap_or("source").to_string()], + source_ref: Some(SourceRef::new(format!("{scope}/{seq}"))), + path_scope: None, + }, + token_count: 80, + seq_in_source: seq as u32, + created_at: ts, + partial_message: false, + }; + upsert_chunks(cfg, std::slice::from_ref(&chunk))?; + + let leaf = LeafRef { + chunk_id: chunk.id, + // >= input_token_budget so each leaf seals into its own L0. + token_count: cfg.tree.input_token_budget, + timestamp: ts, + content: (*content).to_string(), + entities: entities.iter().map(|e| e.to_string()).collect(), + topics: topics.iter().map(|t| t.to_string()).collect(), + score: 0.8, + }; + factory.insert_leaf(cfg, &leaf, &summariser).await?; + } + factory.seal_now(cfg, &summariser).await?; + Ok(()) +} + +async fn seed_trees(root: &std::path::Path) -> anyhow::Result<()> { + let cfg = MemoryConfig::new(root); + + let gmail: &[Message] = &[ + ( + "Kicking off Q3 planning — please send roadmap inputs by Friday.", + &["person:alice", "topic:planning"], + &["topic:planning"], + ), + ( + "Roadmap review moved to Thursday; metrics dashboard added to the deck.", + &["person:alice", "person:bob"], + &["topic:planning"], + ), + ( + "Acme Cloud invoice #4021 — $412 due on the 5th, auto-pay enabled.", + &["org:acme"], + &["topic:billing"], + ), + ( + "Re: budget — engineering headcount approved for two hires.", + &["person:carol"], + &["topic:budget"], + ), + ( + "Offsite logistics: booking the venue for the last week of August.", + &["person:bob"], + &["topic:offsite"], + ), + ( + "Security review scheduled — please rotate the staging credentials.", + &["person:dave", "topic:security"], + &["topic:security"], + ), + ( + "Customer escalation from Globex resolved; postmortem attached.", + &["org:globex"], + &["topic:support"], + ), + ( + "Weekly metrics: signups up 12%, churn flat, NPS at 41.", + &["topic:metrics"], + &["topic:metrics"], + ), + ( + "Design handoff for the memory viewer is ready for review.", + &["person:erin", "topic:design"], + &["topic:design"], + ), + ( + "Reminder: submit expense reports before end of month.", + &["topic:ops"], + &["topic:ops"], + ), + ( + "Re: hiring — onsite loop for the platform role is confirmed.", + &["person:carol"], + &["topic:hiring"], + ), + ]; + + let github: &[Message] = &[ + ( + "PR #882: harden queue and tree concurrency, add busy-timeout.", + &["repo:tinycortex", "topic:concurrency"], + &["topic:concurrency"], + ), + ( + "Issue #66: port OpenHuman engine gaps — entity index and fan-out.", + &["repo:tinycortex"], + &["topic:parity"], + ), + ( + "PR #884: durable KvSkillDocSink for inspectable synced memory.", + &["repo:tinycortex", "topic:sync"], + &["topic:sync"], + ), + ( + "Issue #71: intermittent SQLITE_BUSY under parallel ingest.", + &["topic:bug"], + &["topic:bug"], + ), + ( + "PR #889: Next.js memory viewer reading the workspace server-side.", + &["repo:tinycortex", "topic:viewer"], + &["topic:viewer"], + ), + ( + "Issue #73: document the on-disk layout of the memory tree.", + &["topic:docs"], + &["topic:docs"], + ), + ( + "PR #891: force-directed graph view of the summary tree.", + &["topic:viewer", "topic:graph"], + &["topic:graph"], + ), + ( + "Issue #75: entity co-occurrence edges are not populated on insert.", + &["topic:entities"], + &["topic:entities"], + ), + ( + "PR #893: seed example builds a real tree for demos.", + &["repo:tinycortex"], + &["topic:demo"], + ), + ( + "Issue #77: Composio connect flow for new account ids.", + &["topic:composio"], + &["topic:composio"], + ), + ( + "PR #895: load .env in the composio harness.", + &["topic:composio"], + &["topic:config"], + ), + ]; + + seed_tree(&cfg, SourceKind::Email, "gmail:inbox", "alice", gmail).await?; + seed_tree(&cfg, SourceKind::Chat, "github:tinycortex", "bot", github).await?; + Ok(()) +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let workspace = std::env::var("TINYCORTEX_WORKSPACE") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| { + std::env::temp_dir() + .join("tinycortex-memory-demo") + .to_string_lossy() + .into_owned() + }); + let root = std::path::Path::new(&workspace); + + let doc_count = seed_skill_docs(root).await?; + seed_trees(root).await?; + + println!( + "Seeded {doc_count} skill documents to {} and built a memory tree in {}.", + root.join(SKILL_DOCS_DB).display(), + root.join("memory_tree/chunks.db").display() + ); + println!("Point the viewer at it: TINYCORTEX_WORKSPACE={workspace} npm run dev"); + Ok(()) +} diff --git a/src/memory/sync/composio/connect.rs b/src/memory/sync/composio/connect.rs new file mode 100644 index 0000000..1ea0998 --- /dev/null +++ b/src/memory/sync/composio/connect.rs @@ -0,0 +1,364 @@ +//! Composio v3 login/connect helpers. +//! +//! This module owns the reusable, host-agnostic pieces of the Composio +//! connection flow so a harness (or a server-side host) can drive an OAuth +//! login without re-deriving the wire contract: +//! +//! * a small per-integration **entity-id store** ([`EntityStore`]) that +//! remembers the `user_id` chosen for each toolkit across runs so re-runs +//! reuse the same Composio "entity" instead of orphaning connections; +//! * pure parsers for the connected-account **status** lifecycle; and +//! * thin async wrappers over the three v3 endpoints the connect flow needs. +//! +//! ## Verified v3 endpoints +//! +//! All confirmed against the Composio SDK source (the generated OpenAPI client +//! these SDKs wrap) — — and the v3 API +//! reference at : +//! +//! * `GET /api/v3/auth_configs?toolkit_slug={slug}` — list auth configs; +//! response `{ items: [ { id, toolkit: { slug } } ] }`. +//! (`ts/packages/core/src/models/AuthConfigs.ts`, `authConfigs.types.ts`.) +//! * `POST /api/v3/connected_accounts/link` — create a Composio Connect Link; +//! body `{ auth_config_id, user_id, callback_url? }`, response +//! `{ connected_account_id, redirect_url }`. +//! (`ConnectedAccounts.ts` `link()`, `connected_accounts.py` `link()`.) +//! * `GET /api/v3/connected_accounts/{nanoid}` — poll status; top-level +//! `status` in `INITIALIZING | INITIATED | ACTIVE | EXPIRED | FAILED | +//! REVOKED`. (`connected_accounts.py` `wait_for_connection`.) +//! +//! Authentication is the direct-mode `x-api-key` header, matching +//! [`super::client::ComposioClient`]. No secret is ever logged and error paths +//! discard raw response bodies (they can echo the key back verbatim). + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +/// Connection statuses that will never recover on their own — polling should +/// stop and fail. Mirrors the Composio SDK's `terminalErrorStates` +/// (`FAILED`, `EXPIRED`, `REVOKED`); `INACTIVE` is intentionally excluded +/// because it can transition back to `ACTIVE`. +const TERMINAL_STATUSES: &[&str] = &["FAILED", "EXPIRED", "REVOKED", "DELETED"]; + +/// Generate a fresh Composio entity id (`user_id`) for a new connection. +/// +/// The `tinycortex-` prefix keeps harness-created entities recognisable in the +/// Composio dashboard while the UUID guarantees uniqueness. +pub fn generate_entity_id() -> String { + format!("tinycortex-{}", Uuid::new_v4()) +} + +/// True when a connected-account status string means the account is live and +/// usable for tool execution. Case-insensitive. +pub fn status_is_active(status: &str) -> bool { + status.trim().eq_ignore_ascii_case("ACTIVE") +} + +/// True when a status string is a terminal failure that polling must give up +/// on. Case-insensitive. +pub fn status_is_terminal(status: &str) -> bool { + let status = status.trim(); + TERMINAL_STATUSES + .iter() + .any(|terminal| status.eq_ignore_ascii_case(terminal)) +} + +/// Pull the connected-account `status` out of a get-by-id response. +/// +/// Composio has shipped the status both at the top level and nested under +/// `state.val` / `connectionData.val`; probe the known shapes. +pub fn extract_status(record: &Value) -> Option { + [ + record.get("status"), + record.pointer("/state/val/status"), + record.pointer("/connectionData/val/status"), + record.pointer("/connection_data/val/status"), + ] + .into_iter() + .flatten() + .find_map(Value::as_str) + .map(str::trim) + .filter(|status| !status.is_empty()) + .map(str::to_owned) +} + +/// Extract the OAuth redirect URL from a create-link response. The v3 `/link` +/// endpoint returns a flat `redirect_url`; older shapes nested it under +/// `connectionData.val.redirectUrl`, so probe both. +pub fn extract_redirect_url(record: &Value) -> Option { + [ + record.get("redirect_url"), + record.get("redirectUrl"), + record.pointer("/connectionData/val/redirectUrl"), + record.pointer("/connection_data/val/redirect_url"), + ] + .into_iter() + .flatten() + .find_map(Value::as_str) + .map(str::trim) + .filter(|url| !url.is_empty()) + .map(str::to_owned) +} + +/// Extract the connected-account id from a create-link response. The v3 +/// `/link` endpoint returns `connected_account_id`; legacy `initiate` returned +/// a top-level `id`. +pub fn extract_account_id(record: &Value) -> Option { + ["connected_account_id", "connectedAccountId", "id", "nanoid"] + .iter() + .find_map(|key| record.get(key).and_then(Value::as_str)) + .map(str::trim) + .filter(|id| !id.is_empty()) + .map(str::to_owned) +} + +/// Resolve an auth-config id for `toolkit` from a `GET /auth_configs` response. +/// +/// Prefers an item whose `toolkit.slug` matches (case-insensitively); falls +/// back to the first listed config when the toolkit was already used as a +/// server-side filter and the slug field is shaped differently. +pub fn resolve_auth_config_id(list: &Value, toolkit: &str) -> Option { + let items = list + .pointer("/items") + .and_then(Value::as_array) + .or_else(|| list.get("data").and_then(Value::as_array)) + .or_else(|| list.as_array())?; + + let matches_toolkit = |item: &Value| { + [ + item.pointer("/toolkit/slug"), + item.pointer("/toolkit/name"), + item.get("toolkit"), + ] + .into_iter() + .flatten() + .find_map(Value::as_str) + .map(|slug| slug.trim().eq_ignore_ascii_case(toolkit)) + .unwrap_or(false) + }; + let auth_config_id = |item: &Value| { + ["id", "nanoid"] + .iter() + .find_map(|key| item.get(key).and_then(Value::as_str)) + .map(str::trim) + .filter(|id| !id.is_empty()) + .map(str::to_owned) + }; + + items + .iter() + .find(|item| matches_toolkit(item)) + .and_then(auth_config_id) + .or_else(|| items.iter().find_map(auth_config_id)) +} + +/// A newly-created Composio Connect Link. +#[derive(Debug, Clone)] +pub struct ConnectionLink { + /// The pending connected-account id to poll for `ACTIVE`. + pub connected_account_id: String, + /// The OAuth URL the user must open to complete login, when the scheme is + /// redirect-based. `None` for schemes that activate without a browser step. + pub redirect_url: Option, +} + +/// Persistent, per-toolkit map of the entity id (`user_id`) chosen for each +/// integration. +/// +/// Stored as a small JSON object on disk (e.g. `.composio-harness.json`) so a +/// re-run reuses the same Composio entity instead of creating a fresh — and +/// therefore orphaned — connection every time. Load is best-effort: a missing +/// or corrupt file yields an empty store rather than an error. +#[derive(Debug, Clone)] +pub struct EntityStore { + path: PathBuf, + entries: BTreeMap, +} + +#[derive(Default, Serialize, Deserialize)] +struct EntityStoreFile { + /// toolkit slug -> entity id (`user_id`). + #[serde(default)] + entities: BTreeMap, +} + +impl EntityStore { + /// Load the store from `path`, tolerating a missing or unreadable file. + pub fn load(path: impl Into) -> Self { + let path = path.into(); + let entries = std::fs::read_to_string(&path) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .map(|file| file.entities) + .unwrap_or_default(); + Self { path, entries } + } + + /// The backing file path. + pub fn path(&self) -> &Path { + &self.path + } + + /// The entity id recorded for `toolkit`, if any. + pub fn get(&self, toolkit: &str) -> Option<&str> { + self.entries.get(toolkit).map(String::as_str) + } + + /// Record `entity_id` for `toolkit` in memory (call [`Self::save`] to + /// persist). + pub fn set(&mut self, toolkit: &str, entity_id: impl Into) { + self.entries.insert(toolkit.to_owned(), entity_id.into()); + } + + /// Resolve the entity id to use when connecting `toolkit`, persisting the + /// choice so future runs are stable. + /// + /// Precedence: a value already stored for this toolkit wins; otherwise an + /// explicit `override_id` (e.g. `COMPOSIO_ENTITY_ID`) is adopted; otherwise + /// a fresh id is generated. The resolved id is written back and saved. + pub fn entity_id_for( + &mut self, + toolkit: &str, + override_id: Option<&str>, + ) -> std::io::Result { + if let Some(existing) = self.get(toolkit) { + return Ok(existing.to_owned()); + } + let chosen = override_id + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .unwrap_or_else(generate_entity_id); + self.set(toolkit, chosen.clone()); + self.save()?; + Ok(chosen) + } + + /// Serialize the store to its backing file (pretty JSON). + pub fn save(&self) -> std::io::Result<()> { + let file = EntityStoreFile { + entities: self.entries.clone(), + }; + let json = serde_json::to_string_pretty(&file) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + std::fs::write(&self.path, json) + } +} + +/// `GET /api/v3/auth_configs?toolkit_slug={toolkit}` — list auth configs, +/// optionally filtered to one toolkit. Returns the decoded JSON body so the +/// caller can resolve an id via [`resolve_auth_config_id`]. +pub async fn list_auth_configs( + http: &reqwest::Client, + base_url: &str, + api_key: &str, + toolkit: Option<&str>, +) -> anyhow::Result { + let mut request = http + .get(format!("{}/auth_configs", base_url.trim_end_matches('/'))) + .header("x-api-key", api_key); + if let Some(toolkit) = toolkit { + request = request.query(&[("toolkit_slug", toolkit)]); + } + let response = request + .send() + .await + .map_err(|error| anyhow::anyhow!("auth_configs request failed: {error}"))?; + let status = response.status(); + if !status.is_success() { + // Never echo the body; it can contain the key back verbatim. + let _ = response.bytes().await; + anyhow::bail!("auth_configs returned HTTP {status}"); + } + response + .json() + .await + .map_err(|error| anyhow::anyhow!("auth_configs decode failed: {error}")) +} + +/// `POST /api/v3/connected_accounts/link` — create a Composio Connect Link for +/// `auth_config_id` scoped to `user_id`. Returns the pending account id plus an +/// optional OAuth redirect URL. +pub async fn create_connection_link( + http: &reqwest::Client, + base_url: &str, + api_key: &str, + auth_config_id: &str, + user_id: &str, + callback_url: Option<&str>, +) -> anyhow::Result { + let mut body = serde_json::json!({ + "auth_config_id": auth_config_id, + "user_id": user_id, + }); + if let Some(callback_url) = callback_url + .map(str::trim) + .filter(|value| !value.is_empty()) + { + body["callback_url"] = serde_json::json!(callback_url); + } + let response = http + .post(format!( + "{}/connected_accounts/link", + base_url.trim_end_matches('/') + )) + .header("x-api-key", api_key) + .json(&body) + .send() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts/link request failed: {error}"))?; + let status = response.status(); + if !status.is_success() { + let _ = response.bytes().await; + anyhow::bail!("connected_accounts/link returned HTTP {status}"); + } + let record: Value = response + .json() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts/link decode failed: {error}"))?; + let connected_account_id = extract_account_id(&record).ok_or_else(|| { + anyhow::anyhow!("connected_accounts/link response missing a connected account id") + })?; + Ok(ConnectionLink { + connected_account_id, + redirect_url: extract_redirect_url(&record), + }) +} + +/// `GET /api/v3/connected_accounts/{account_id}` — fetch the current status of +/// a (possibly pending) connected account. Returns `None` if the response had +/// no recognisable status field. +pub async fn get_connection_status( + http: &reqwest::Client, + base_url: &str, + api_key: &str, + account_id: &str, +) -> anyhow::Result> { + let response = http + .get(format!( + "{}/connected_accounts/{account_id}", + base_url.trim_end_matches('/') + )) + .header("x-api-key", api_key) + .send() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts/{{id}} request failed: {error}"))?; + let status = response.status(); + if !status.is_success() { + let _ = response.bytes().await; + anyhow::bail!("connected_accounts/{{id}} returned HTTP {status}"); + } + let record: Value = response + .json() + .await + .map_err(|error| anyhow::anyhow!("connected_accounts/{{id}} decode failed: {error}"))?; + Ok(extract_status(&record)) +} + +#[cfg(test)] +#[path = "connect_tests.rs"] +mod tests; diff --git a/src/memory/sync/composio/connect_tests.rs b/src/memory/sync/composio/connect_tests.rs new file mode 100644 index 0000000..e24fdc8 --- /dev/null +++ b/src/memory/sync/composio/connect_tests.rs @@ -0,0 +1,148 @@ +//! Unit tests for the pure Composio connect helpers: entity-id persistence, +//! status classification, and response-field extraction. No network I/O. + +use super::*; +use serde_json::json; + +#[test] +fn generated_entity_id_is_prefixed_and_unique() { + let a = generate_entity_id(); + let b = generate_entity_id(); + assert!(a.starts_with("tinycortex-"), "unexpected id: {a}"); + assert_ne!(a, b, "two generated ids must differ"); +} + +#[test] +fn status_active_is_case_insensitive() { + assert!(status_is_active("ACTIVE")); + assert!(status_is_active("active")); + assert!(status_is_active(" Active ")); + assert!(!status_is_active("INITIATED")); + assert!(!status_is_active("FAILED")); +} + +#[test] +fn status_terminal_matches_failure_states_only() { + for terminal in ["FAILED", "expired", "Revoked", "DELETED"] { + assert!( + status_is_terminal(terminal), + "{terminal} should be terminal" + ); + } + for live in ["ACTIVE", "INITIATED", "INITIALIZING", "INACTIVE"] { + assert!(!status_is_terminal(live), "{live} should not be terminal"); + } +} + +#[test] +fn extract_status_probes_top_level_and_nested() { + assert_eq!( + extract_status(&json!({"status": "ACTIVE"})).as_deref(), + Some("ACTIVE") + ); + assert_eq!( + extract_status(&json!({"state": {"val": {"status": "INITIATED"}}})).as_deref(), + Some("INITIATED") + ); + assert_eq!( + extract_status(&json!({"connectionData": {"val": {"status": "EXPIRED"}}})).as_deref(), + Some("EXPIRED") + ); + assert_eq!(extract_status(&json!({"other": 1})), None); +} + +#[test] +fn extract_link_fields_from_v3_shape() { + let response = json!({ + "connected_account_id": "ca_abc123", + "redirect_url": "https://backend.composio.dev/oauth/start?token=xyz", + "link_token": "lt_123", + }); + assert_eq!(extract_account_id(&response).as_deref(), Some("ca_abc123")); + assert_eq!( + extract_redirect_url(&response).as_deref(), + Some("https://backend.composio.dev/oauth/start?token=xyz") + ); +} + +#[test] +fn extract_link_fields_tolerates_legacy_shape() { + let response = json!({ + "id": "ca_legacy", + "connectionData": {"val": {"status": "INITIATED", "redirectUrl": "https://x/y"}}, + }); + assert_eq!(extract_account_id(&response).as_deref(), Some("ca_legacy")); + assert_eq!( + extract_redirect_url(&response).as_deref(), + Some("https://x/y") + ); +} + +#[test] +fn resolve_auth_config_prefers_matching_toolkit() { + let list = json!({ + "items": [ + {"id": "ac_github", "toolkit": {"slug": "github"}}, + {"id": "ac_gmail", "toolkit": {"slug": "gmail"}}, + ] + }); + assert_eq!( + resolve_auth_config_id(&list, "gmail").as_deref(), + Some("ac_gmail") + ); + assert_eq!( + resolve_auth_config_id(&list, "github").as_deref(), + Some("ac_github") + ); +} + +#[test] +fn resolve_auth_config_falls_back_to_first_when_no_slug_match() { + // Server already filtered by toolkit_slug; items may not echo a slug shape + // we recognise, so fall back to the first listed config. + let list = json!({ "items": [ {"id": "ac_only"} ] }); + assert_eq!( + resolve_auth_config_id(&list, "gmail").as_deref(), + Some("ac_only") + ); + assert_eq!(resolve_auth_config_id(&json!({"items": []}), "gmail"), None); +} + +#[test] +fn entity_store_round_trips_and_reuses_ids() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(".composio-harness.json"); + + let mut store = EntityStore::load(&path); + assert_eq!(store.get("gmail"), None); + + // First resolve for gmail with an explicit override adopts + persists it. + let gmail_id = store.entity_id_for("gmail", Some("my-entity")).unwrap(); + assert_eq!(gmail_id, "my-entity"); + + // A second toolkit with no override generates a fresh id. + let github_id = store.entity_id_for("github", None).unwrap(); + assert!(github_id.starts_with("tinycortex-")); + assert_ne!(github_id, gmail_id); + + // Reload from disk: both ids survived and are stable on re-resolve. + let mut reloaded = EntityStore::load(&path); + assert_eq!(reloaded.get("gmail"), Some("my-entity")); + assert_eq!(reloaded.get("github"), Some(github_id.as_str())); + // Stored value wins even if a different override is passed on re-run. + assert_eq!( + reloaded.entity_id_for("gmail", Some("different")).unwrap(), + "my-entity" + ); +} + +#[test] +fn entity_store_load_tolerates_missing_and_corrupt_files() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("nope.json"); + assert_eq!(EntityStore::load(&missing).get("gmail"), None); + + let corrupt = dir.path().join("corrupt.json"); + std::fs::write(&corrupt, "{ not json ").unwrap(); + assert_eq!(EntityStore::load(&corrupt).get("gmail"), None); +} diff --git a/src/memory/sync/composio/mod.rs b/src/memory/sync/composio/mod.rs index c1ed56b..1af0a74 100644 --- a/src/memory/sync/composio/mod.rs +++ b/src/memory/sync/composio/mod.rs @@ -1,11 +1,16 @@ //! Composio-backed synchronization. pub mod client; +pub mod connect; pub mod gmail; pub mod orchestrator; pub mod providers; pub use client::{ActionExecutor, ComposioClient, ExecuteError, ExecuteResponse}; +pub use connect::{ + create_connection_link, generate_entity_id, get_connection_status, list_auth_configs, + resolve_auth_config_id, status_is_active, status_is_terminal, ConnectionLink, EntityStore, +}; pub use gmail::GmailSyncPipeline; pub use orchestrator::{run_incremental_sync, IncrementalSource, PageFetch, SyncItem, SyncScope}; pub use providers::{ diff --git a/src/memory/sync/mod.rs b/src/memory/sync/mod.rs index c27f3ad..f9cf60f 100644 --- a/src/memory/sync/mod.rs +++ b/src/memory/sync/mod.rs @@ -5,6 +5,7 @@ pub mod composio; pub mod dispatcher; pub mod github; pub mod periodic; +pub mod persist; pub mod rebuild; pub mod state; pub mod status; @@ -15,12 +16,15 @@ pub use audit::{ append_audit_entry, estimate_cost_usd, read_audit_log, RealCostAccumulator, SyncAuditEntry, }; pub use composio::{ - ClickUpSyncPipeline, ComposioClient, GitHubSyncPipeline, GmailSyncPipeline, LinearSyncPipeline, - NotionSyncPipeline, SlackSearchBackfillPipeline, SlackSyncPipeline, + create_connection_link, generate_entity_id, get_connection_status, list_auth_configs, + resolve_auth_config_id, status_is_active, status_is_terminal, ClickUpSyncPipeline, + ComposioClient, ConnectionLink, EntityStore, GitHubSyncPipeline, GmailSyncPipeline, + LinearSyncPipeline, NotionSyncPipeline, SlackSearchBackfillPipeline, SlackSyncPipeline, }; pub use dispatcher::{SyncDispatcher, SyncRunResult}; pub use github::GithubRepoSyncPipeline; pub use periodic::{due_workspace_sources, effective_interval_secs, DEFAULT_SYNC_INTERVAL_SECS}; +pub use persist::{KvSkillDocSink, SKILLDOC_NS_PREFIX, SKILL_DOCS_DB}; pub use rebuild::{ needs_rebuild, raw_coverage, rebuild_tree_from_raw, rebuild_tree_from_raw_with_audit, RawCoverage, RawFileRef, RebuildOutcome, diff --git a/src/memory/sync/persist.rs b/src/memory/sync/persist.rs new file mode 100644 index 0000000..dad0b7d --- /dev/null +++ b/src/memory/sync/persist.rs @@ -0,0 +1,112 @@ +//! Durable host sinks for live synchronization. +//! +//! Sync pipelines hand [`SkillDocument`]s to a host-owned [`SkillDocSink`] (see +//! [`crate::memory::sync::traits`]); the crate otherwise ships only in-memory +//! sinks for tests, so nothing a sync run ingests survives the process. That is +//! the right default for a library, but it makes a sync run impossible to +//! observe after the fact. +//! +//! [`KvSkillDocSink`] closes that gap with a durable, inspectable reference +//! sink backed by the [`KvStore`]. Every document becomes one `kv_namespace` +//! row, so a host — or a debug viewer — can browse exactly what a sync run +//! ingested without re-running it (and without a live Composio key). +//! +//! ## Layout +//! +//! - namespace: `skilldoc:` (the toolkit slug every +//! provider sets, e.g. `skilldoc:gmail`) +//! - key: `document_id` (e.g. `gmail:18f2…`) +//! - value: the full JSON-serialised [`SkillDocument`] +//! +//! One row per document keeps deletes O(1) and lets a reader enumerate a +//! toolkit's documents with a single namespace scan +//! (`SELECT … WHERE namespace = 'skilldoc:gmail'`). +//! +//! ## Safety +//! +//! Writes go through [`KvStore::set_namespace`], which **sanitizes the stored +//! value** (redacting secrets/PII) before it is persisted. The persisted +//! documents therefore match the scrubbed content the rest of the memory system +//! would hold — a debug viewer over this store never surfaces raw credentials. +//! The namespace/key are derived from safe slugs and ids; if a provider ever +//! emits a `document_id` that itself looks like a secret or personal identifier +//! the underlying store rejects the write, and the error surfaces here rather +//! than being silently dropped. + +use std::path::Path; +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::memory::store::KvStore; +use crate::memory::sync::traits::{SkillDocSink, SkillDocument}; + +/// KV namespace prefix under which skill documents are stored. +pub const SKILLDOC_NS_PREFIX: &str = "skilldoc:"; + +/// Canonical on-disk path for the skill-document KV store, relative to a +/// workspace root. Kept alongside the memory tree so all sync artifacts for a +/// workspace live under `memory_tree/`. +pub const SKILL_DOCS_DB: &str = "memory_tree/skill_docs.db"; + +/// Durable [`SkillDocSink`] that persists every [`SkillDocument`] as one row in +/// a [`KvStore`], so a host or debug viewer can inspect ingested memory. +/// +/// The sink is cheap to clone-share (`Arc` inside); the store +/// serializes its own writes behind a mutex, so concurrent `store`/`delete` +/// calls are safe. +pub struct KvSkillDocSink { + kv: Arc, +} + +impl KvSkillDocSink { + /// Wrap an already-open [`KvStore`]. + pub fn new(kv: Arc) -> Self { + Self { kv } + } + + /// Open (or create) the canonical skill-document store under `workspace` + /// (`/memory_tree/skill_docs.db`). Parent directories are + /// created as needed by the underlying [`KvStore::open`]. + pub fn open_in_workspace(workspace: &Path) -> anyhow::Result { + let db_path = workspace.join(SKILL_DOCS_DB); + let kv = KvStore::open(&db_path)?; + Ok(Self::new(Arc::new(kv))) + } + + /// Borrow the backing store (e.g. to read documents back for inspection). + pub fn store_handle(&self) -> &Arc { + &self.kv + } + + /// KV namespace a given skill id is stored under. + pub fn namespace_for(namespace_skill_id: &str) -> String { + format!("{SKILLDOC_NS_PREFIX}{namespace_skill_id}") + } +} + +#[async_trait] +impl SkillDocSink for KvSkillDocSink { + async fn store(&self, document: SkillDocument) -> anyhow::Result<()> { + let namespace = Self::namespace_for(&document.namespace_skill_id); + let key = document.document_id.clone(); + let value = serde_json::to_value(&document) + .map_err(|error| anyhow::anyhow!("serialize skill doc {key}: {error}"))?; + self.kv + .set_namespace(&namespace, &key, &value) + .map_err(|error| anyhow::anyhow!("persist skill doc {key}: {error}"))?; + Ok(()) + } + + async fn delete(&self, namespace_skill_id: &str, document_id: &str) -> anyhow::Result<()> { + let namespace = Self::namespace_for(namespace_skill_id); + self.kv + .delete_namespace(&namespace, document_id) + .map_err(|error| anyhow::anyhow!("delete skill doc {document_id}: {error}"))?; + Ok(()) + } +} + +#[cfg(test)] +#[path = "persist_tests.rs"] +mod tests; diff --git a/src/memory/sync/persist_tests.rs b/src/memory/sync/persist_tests.rs new file mode 100644 index 0000000..73d3258 --- /dev/null +++ b/src/memory/sync/persist_tests.rs @@ -0,0 +1,119 @@ +//! Unit tests for [`KvSkillDocSink`]. + +use std::sync::Arc; + +use serde_json::json; + +use super::*; +use crate::memory::store::KvStore; +use crate::memory::sync::traits::SkillDocument; + +fn doc(toolkit: &str, id: &str, content: &str) -> SkillDocument { + SkillDocument { + namespace_skill_id: toolkit.into(), + connection_id: "conn_123".into(), + document_id: format!("{toolkit}:{id}"), + title: format!("doc {id}"), + content: content.into(), + toolkit: toolkit.into(), + metadata: json!({ "taint": "external_sync", "provider_id": id }), + } +} + +fn sink() -> (KvSkillDocSink, Arc) { + let kv = Arc::new(KvStore::open_in_memory().expect("open kv")); + (KvSkillDocSink::new(kv.clone()), kv) +} + +#[tokio::test] +async fn store_persists_document_under_toolkit_namespace() { + let (sink, kv) = sink(); + sink.store(doc("gmail", "abc", "hello world")) + .await + .expect("store"); + + let stored = kv + .get_namespace(&KvSkillDocSink::namespace_for("gmail"), "gmail:abc") + .expect("read") + .expect("present"); + assert_eq!(stored["title"], "doc abc"); + assert_eq!(stored["toolkit"], "gmail"); + assert_eq!(stored["content"], "hello world"); + assert_eq!(stored["metadata"]["taint"], "external_sync"); +} + +#[tokio::test] +async fn documents_are_grouped_by_toolkit() { + let (sink, kv) = sink(); + sink.store(doc("gmail", "a", "one")).await.unwrap(); + sink.store(doc("gmail", "b", "two")).await.unwrap(); + sink.store(doc("github", "c", "three")).await.unwrap(); + + let gmail = kv + .records_namespace(&KvSkillDocSink::namespace_for("gmail")) + .expect("list gmail"); + let github = kv + .records_namespace(&KvSkillDocSink::namespace_for("github")) + .expect("list github"); + assert_eq!(gmail.len(), 2); + assert_eq!(github.len(), 1); +} + +#[tokio::test] +async fn store_is_idempotent_upsert() { + let (sink, kv) = sink(); + sink.store(doc("gmail", "a", "v1")).await.unwrap(); + sink.store(doc("gmail", "a", "v2")).await.unwrap(); + + let records = kv + .records_namespace(&KvSkillDocSink::namespace_for("gmail")) + .expect("list"); + assert_eq!(records.len(), 1, "same document_id upserts in place"); + let stored = kv + .get_namespace(&KvSkillDocSink::namespace_for("gmail"), "gmail:a") + .unwrap() + .unwrap(); + assert_eq!(stored["content"], "v2"); +} + +#[tokio::test] +async fn delete_removes_only_the_target() { + let (sink, kv) = sink(); + sink.store(doc("gmail", "a", "one")).await.unwrap(); + sink.store(doc("gmail", "b", "two")).await.unwrap(); + + sink.delete("gmail", "gmail:a").await.expect("delete"); + + assert!(kv + .get_namespace(&KvSkillDocSink::namespace_for("gmail"), "gmail:a") + .unwrap() + .is_none()); + assert!(kv + .get_namespace(&KvSkillDocSink::namespace_for("gmail"), "gmail:b") + .unwrap() + .is_some()); +} + +#[tokio::test] +async fn stored_value_is_sanitized() { + // The KV layer scrubs secrets from the persisted value, so a debug viewer + // over this store never surfaces raw credentials. + let (sink, kv) = sink(); + sink.store(doc( + "gmail", + "secretful", + "token sk-ABCDEF0123456789ABCDEF0123456789 end", + )) + .await + .unwrap(); + + let stored = kv + .get_namespace(&KvSkillDocSink::namespace_for("gmail"), "gmail:secretful") + .unwrap() + .unwrap(); + let content = stored["content"].as_str().unwrap(); + assert!( + !content.contains("sk-ABCDEF0123456789ABCDEF0123456789"), + "raw secret should be redacted before persistence, got: {content}" + ); +} diff --git a/viewer/.gitignore b/viewer/.gitignore new file mode 100644 index 0000000..f068354 --- /dev/null +++ b/viewer/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +.next/ +.shots/ +next-env.d.ts +*.tsbuildinfo +.DS_Store diff --git a/viewer/README.md b/viewer/README.md new file mode 100644 index 0000000..f818242 --- /dev/null +++ b/viewer/README.md @@ -0,0 +1,48 @@ +# TinyCortex Memory Viewer + +A local, read-only debug UI for inspecting a TinyCortex memory workspace. It is +a small Next.js app whose **server-side** code opens the workspace's SQLite +stores and reads the JSON/markdown artifacts a sync run leaves behind — nothing +touches the network, and the database is opened read-only. + +## What it shows + +- **Overview** — which stores exist on disk, document/toolkit/chunk/entity counts. +- **Skill docs** — every `SkillDocument` sync ingested (secrets/PII scrubbed on + write by the KV store), filterable by toolkit and searchable, with a detail + view (content + metadata + raw provider payload). +- **Memory tree** — chunks from `memory_tree/chunks.db` (present after a full + ingest pass). +- **Entities** — canonical entities under `memory_tree/content/entities/`. +- **Sync runs** — the last run's per-toolkit results and event stream from + `sync_manifest.json`. + +## Run it + +```sh +# 1. Produce a workspace to inspect. Either seed a demo (no API key needed): +TINYCORTEX_WORKSPACE=/tmp/tinycortex-demo \ + cargo run --example seed_memory --features sync + +# …or run a real Composio sync into a workspace: +TINYCORTEX_WORKSPACE=/tmp/tinycortex-demo \ + cargo run --example composio_harness --features sync + +# 2. Point the viewer at the same workspace and start it. +cd viewer +npm install +TINYCORTEX_WORKSPACE=/tmp/tinycortex-demo npm run dev +# → http://localhost:4319 +``` + +The viewer reads whatever is on disk each request, so re-running a sync and +refreshing the page shows the new state. + +## Environment + +- `TINYCORTEX_WORKSPACE` — path to the workspace to inspect (required). + +## Stack + +Next.js (App Router, server components), `better-sqlite3` for read-only SQLite +access. No client-side data fetching; all reads happen on the server. diff --git a/viewer/app/docs/[toolkit]/[id]/page.tsx b/viewer/app/docs/[toolkit]/[id]/page.tsx new file mode 100644 index 0000000..fca0c81 --- /dev/null +++ b/viewer/app/docs/[toolkit]/[id]/page.tsx @@ -0,0 +1,92 @@ +import { Fragment } from "react"; +import Link from "next/link"; +import { getSkillDoc, workspacePath } from "@/lib/memory"; +import { Empty, NoWorkspace, secondsTs } from "@/lib/ui"; + +export default async function DocDetailPage({ + params, +}: { + params: Promise<{ toolkit: string; id: string }>; +}) { + const ws = workspacePath(); + const { toolkit, id } = await params; + const documentId = decodeURIComponent(id); + const tk = decodeURIComponent(toolkit); + + if (!ws) { + return ( + <> +

Document

+ + + ); + } + + const doc = getSkillDoc(ws, tk, documentId); + if (!doc) { + return ( + <> + + ← Skill docs + +

Document not found

+ + + {tk}:{documentId} + {" "} + is not in this workspace. + + + ); + } + + const meta = Object.entries(doc.metadata).filter(([k]) => k !== "raw"); + const raw = (doc.metadata as Record).raw; + + return ( + <> + + ← {doc.toolkit} docs + +

{doc.title}

+

+ {doc.toolkit}{" "} + {doc.documentId} +

+ +
+

Metadata

+
+
connection
+
{doc.connectionId || "—"}
+
skill id
+
{doc.namespaceSkillId}
+
updated
+
{secondsTs(doc.updatedAt)}
+ {meta.map(([k, v]) => ( + +
{k}
+
{typeof v === "string" ? v : JSON.stringify(v)}
+ + ))} +
+
+ +
+

Content

+
+ 
{doc.content || "(empty)"}
+
+
+ + {raw !== undefined && ( +
+

Raw provider payload

+
+ 
{JSON.stringify(raw, null, 2)}
+
+
+ )} + + ); +} diff --git a/viewer/app/docs/page.tsx b/viewer/app/docs/page.tsx new file mode 100644 index 0000000..652ce86 --- /dev/null +++ b/viewer/app/docs/page.tsx @@ -0,0 +1,108 @@ +import Link from "next/link"; +import { listSkillDocs, skillDocCountsByToolkit, workspacePath } from "@/lib/memory"; +import { Empty, NoWorkspace, secondsTs } from "@/lib/ui"; + +export default async function DocsPage({ + searchParams, +}: { + searchParams: Promise<{ toolkit?: string; q?: string }>; +}) { + const ws = workspacePath(); + if (!ws) { + return ( + <> +

Skill documents

+ + + ); + } + + const { toolkit, q } = await searchParams; + const all = listSkillDocs(ws); + const toolkits = skillDocCountsByToolkit(all); + const needle = (q ?? "").trim().toLowerCase(); + + const docs = all.filter((d) => { + if (toolkit && d.toolkit !== toolkit) return false; + if (needle && !`${d.title} ${d.content}`.toLowerCase().includes(needle)) return false; + return true; + }); + + return ( + <> +

Skill documents

+

+ {docs.length} of {all.length} documents ingested by sync (secrets/PII scrubbed on write). +

+ +
+ {toolkit && } + +
+ +
+ + all + + {toolkits.map((t) => ( + + {t.toolkit} ({t.count}) + + ))} +
+ + {docs.length === 0 ? ( + No documents match. + ) : ( +
+
+ + + + + + + + + + + + {docs.map((d) => ( + + + + + + + + ))} + +
ToolkitTitleDocument IDTaintUpdated
+ {d.toolkit} + + + {d.title} + + {d.documentId} + {String((d.metadata as Record).taint ?? "—")} + {secondsTs(d.updatedAt)}
+
+
+ )} + + ); +} diff --git a/viewer/app/entities/page.tsx b/viewer/app/entities/page.tsx new file mode 100644 index 0000000..f5d1ad3 --- /dev/null +++ b/viewer/app/entities/page.tsx @@ -0,0 +1,57 @@ +import { Fragment } from "react"; +import { listEntities, workspacePath } from "@/lib/memory"; +import { Empty, NoWorkspace } from "@/lib/ui"; + +export default function EntitiesPage() { + const ws = workspacePath(); + if (!ws) { + return ( + <> +

Entities

+ + + ); + } + + const entities = listEntities(ws); + if (entities.length === 0) { + return ( + <> +

Entities

+

Canonical entities extracted into the memory tree.

+ + No entities under memory_tree/content/entities/ yet. + + + ); + } + + return ( + <> +

Entities

+

{entities.length} entity files.

+ {entities.map((e) => ( +
+

+ {e.kind} · {e.file} +

+ {Object.keys(e.frontMatter).length > 0 && ( +
+ {Object.entries(e.frontMatter).map(([k, v]) => ( + +
{k}
+
{v}
+ + ))} +
+ )} + {e.body.trim() && ( +
+ 
{e.body.trim()}
+
+ )} +
+ ))} + + ); +} diff --git a/viewer/app/globals.css b/viewer/app/globals.css new file mode 100644 index 0000000..60e815c --- /dev/null +++ b/viewer/app/globals.css @@ -0,0 +1,469 @@ +:root { + --bg: #f7f7f8; + --panel: #ffffff; + --border: #e4e4e7; + --text: #18181b; + --muted: #71717a; + --accent: #4f46e5; + --accent-soft: #eef2ff; + --ok: #16794a; + --ok-soft: #e7f6ee; + --fail: #b42318; + --fail-soft: #fdeceb; + --code: #f4f4f5; + --shadow: 0 1px 2px rgba(0, 0, 0, 0.04), 0 1px 3px rgba(0, 0, 0, 0.06); + --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; +} + +@media (prefers-color-scheme: dark) { + :root { + --bg: #0b0b0d; + --panel: #161619; + --border: #27272a; + --text: #ededf0; + --muted: #8a8a93; + --accent: #8b87f5; + --accent-soft: #1e1b34; + --ok: #4ade80; + --ok-soft: #10241a; + --fail: #f87171; + --fail-soft: #2a1414; + --code: #1d1d21; + --shadow: none; + } +} + +* { + box-sizing: border-box; +} + +html, +body { + margin: 0; + padding: 0; +} + +body { + background: var(--bg); + color: var(--text); + font-family: + -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + font-size: 14px; + line-height: 1.5; + -webkit-font-smoothing: antialiased; +} + +a { + color: inherit; + text-decoration: none; +} + +.shell { + display: grid; + grid-template-columns: 220px 1fr; + min-height: 100vh; +} + +.sidebar { + border-right: 1px solid var(--border); + padding: 20px 14px; + background: var(--panel); + position: sticky; + top: 0; + height: 100vh; +} + +.brand { + font-weight: 650; + font-size: 15px; + letter-spacing: -0.01em; + padding: 0 8px 4px; +} + +.brand small { + display: block; + font-weight: 400; + color: var(--muted); + font-size: 11px; + margin-top: 2px; +} + +.nav { + margin-top: 18px; + display: flex; + flex-direction: column; + gap: 2px; +} + +.nav a { + padding: 7px 10px; + border-radius: 7px; + color: var(--muted); + font-weight: 500; + display: flex; + justify-content: space-between; + align-items: center; +} + +.nav a:hover { + background: var(--code); + color: var(--text); +} + +.nav a.active { + background: var(--accent-soft); + color: var(--accent); +} + +.nav .pill { + font-size: 11px; + font-variant-numeric: tabular-nums; + background: var(--code); + color: var(--muted); + border-radius: 999px; + padding: 1px 7px; +} + +.main { + padding: 28px 32px 64px; + max-width: 1100px; +} + +h1 { + font-size: 20px; + margin: 0 0 4px; + letter-spacing: -0.02em; +} + +.subtitle { + color: var(--muted); + margin: 0 0 24px; +} + +.grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); + gap: 12px; + margin-bottom: 24px; +} + +.card { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 10px; + padding: 14px 16px; + box-shadow: var(--shadow); +} + +.card .k { + color: var(--muted); + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.card .v { + font-size: 26px; + font-weight: 650; + font-variant-numeric: tabular-nums; + margin-top: 4px; +} + +.panel { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 10px; + box-shadow: var(--shadow); + overflow: hidden; + margin-bottom: 20px; +} + +.panel h2 { + font-size: 13px; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--muted); + margin: 0; + padding: 12px 16px; + border-bottom: 1px solid var(--border); +} + +.tablewrap { + overflow-x: auto; +} + +table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} + +th, +td { + text-align: left; + padding: 9px 16px; + border-bottom: 1px solid var(--border); + vertical-align: top; +} + +th { + color: var(--muted); + font-weight: 500; + font-size: 12px; + white-space: nowrap; +} + +tbody tr:last-child td { + border-bottom: none; +} + +tbody tr:hover { + background: var(--code); +} + +.mono { + font-family: var(--mono); + font-size: 12px; +} + +.muted { + color: var(--muted); +} + +.tag { + display: inline-block; + background: var(--code); + color: var(--muted); + border-radius: 6px; + padding: 1px 7px; + font-size: 11px; + margin: 0 4px 4px 0; + font-family: var(--mono); +} + +.badge { + display: inline-block; + border-radius: 999px; + padding: 1px 9px; + font-size: 11px; + font-weight: 600; +} + +.badge.ok { + background: var(--ok-soft); + color: var(--ok); +} + +.badge.fail { + background: var(--fail-soft); + color: var(--fail); +} + +.badge.toolkit { + background: var(--accent-soft); + color: var(--accent); + font-family: var(--mono); + font-weight: 500; +} + +.rowlink { + color: var(--accent); + font-weight: 500; +} + +.filters { + display: flex; + gap: 8px; + flex-wrap: wrap; + margin-bottom: 16px; +} + +.filters input, +.filters a { + font: inherit; + padding: 7px 11px; + border-radius: 8px; + border: 1px solid var(--border); + background: var(--panel); + color: var(--text); +} + +.filters a.chip { + color: var(--muted); +} + +.filters a.chip.active { + background: var(--accent-soft); + border-color: var(--accent); + color: var(--accent); +} + +.filters input { + min-width: 220px; +} + +pre.content { + background: var(--code); + border: 1px solid var(--border); + border-radius: 8px; + padding: 14px 16px; + white-space: pre-wrap; + word-break: break-word; + font-family: var(--mono); + font-size: 12.5px; + margin: 0; +} + +.kv { + display: grid; + grid-template-columns: 140px 1fr; + gap: 6px 14px; + padding: 14px 16px; + font-size: 13px; +} + +.kv dt { + color: var(--muted); +} + +.kv dd { + margin: 0; + font-family: var(--mono); + font-size: 12.5px; + word-break: break-word; +} + +.empty { + padding: 40px 24px; + text-align: center; + color: var(--muted); +} + +.empty code { + background: var(--code); + padding: 2px 6px; + border-radius: 5px; + font-family: var(--mono); +} + +.back { + color: var(--muted); + font-size: 13px; + margin-bottom: 14px; + display: inline-block; +} + +.back:hover { + color: var(--accent); +} + +/* ── Memory graph ─────────────────────────────────────────────────────────── */ + +:root { + --graph-edge: rgba(120, 120, 135, 0.35); +} + +@media (prefers-color-scheme: dark) { + :root { + --graph-edge: rgba(150, 150, 170, 0.22); + } +} + +.graphwrap { + position: relative; + height: calc(100vh - 150px); + min-height: 460px; + background: var(--panel); + border: 1px solid var(--border); + border-radius: 12px; + box-shadow: var(--shadow); + overflow: hidden; + touch-action: none; + cursor: grab; +} + +.graphwrap svg { + display: block; +} + +.graphwrap svg text { + user-select: none; +} + +.graphtip { + position: absolute; + top: 12px; + left: 12px; + max-width: 340px; + background: var(--panel); + border: 1px solid var(--border); + border-radius: 9px; + padding: 9px 12px; + box-shadow: var(--shadow); + pointer-events: none; +} + +.graphtip-kind { + font-family: var(--mono); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--accent); + margin-bottom: 3px; +} + +.graphtip-label { + font-size: 12.5px; + line-height: 1.35; + max-height: 6.5em; + overflow: hidden; +} + +.graphfit { + position: absolute; + top: 12px; + right: 12px; + font: inherit; + font-size: 12px; + font-weight: 500; + color: var(--text); + background: var(--panel); + border: 1px solid var(--border); + border-radius: 8px; + padding: 6px 14px; + box-shadow: var(--shadow); + cursor: pointer; +} + +.graphfit:hover { + border-color: var(--accent); + color: var(--accent); +} + +.graphlegend { + position: absolute; + bottom: 12px; + right: 12px; + display: flex; + gap: 14px; + background: var(--panel); + border: 1px solid var(--border); + border-radius: 999px; + padding: 6px 14px; + box-shadow: var(--shadow); + font-size: 12px; + color: var(--muted); +} + +.graphlegend span { + display: inline-flex; + align-items: center; + gap: 5px; +} + +.graphlegend i { + width: 9px; + height: 9px; + border-radius: 50%; + display: inline-block; +} diff --git a/viewer/app/graph/ForceGraph.tsx b/viewer/app/graph/ForceGraph.tsx new file mode 100644 index 0000000..b2e8293 --- /dev/null +++ b/viewer/app/graph/ForceGraph.tsx @@ -0,0 +1,344 @@ +"use client"; + +import { useEffect, useMemo, useRef, useState } from "react"; +import { + forceCenter, + forceCollide, + forceLink, + forceManyBody, + forceSimulation, + type Simulation, +} from "d3-force"; +import type { GraphEdge, GraphNode, GraphNodeKind } from "@/lib/memory"; + +type SimNode = GraphNode & { + x: number; + y: number; + vx?: number; + vy?: number; + fx?: number | null; + fy?: number | null; +}; +type SimLink = { source: SimNode | string; target: SimNode | string }; + +const LEVEL_PALETTE = [ + "#7C3AED", + "#4A83DD", + "#1FB6C7", + "#34C77B", + "#E8A653", + "#E0654A", + "#C026D3", +]; + +function color(node: GraphNode): string { + switch (node.kind) { + case "root": + return "#8B5CF6"; + case "source": + return "#F97316"; + case "chunk": + return "#94A3B8"; + case "summary": + return LEVEL_PALETTE[(node.level ?? 0) % LEVEL_PALETTE.length]; + } +} + +function radius(node: GraphNode): number { + switch (node.kind) { + case "root": + return 18; + case "source": + return 14; + case "chunk": + return 3.5; + case "summary": + return Math.min(5 + (node.level ?? 0) * 2.5, 13); + } +} + +const LABEL_KINDS: Set = new Set(["root", "source"]); + +export function ForceGraph({ nodes, edges }: { nodes: GraphNode[]; edges: GraphEdge[] }) { + const wrapRef = useRef(null); + const [size, setSize] = useState({ w: 900, h: 620 }); + const [, setTick] = useState(0); + const [view, setView] = useState({ x: 0, y: 0, k: 0.85 }); + const [hover, setHover] = useState(null); + const [mounted, setMounted] = useState(false); + const simRef = useRef | null>(null); + const dragRef = useRef<{ id: string } | null>(null); + const panRef = useRef<{ x: number; y: number; vx: number; vy: number } | null>(null); + const sizeRef = useRef(size); + const fittedRef = useRef(false); + const fitRef = useRef<() => void>(() => {}); + + // Client-only render gate: node positions are computed by the simulation, so + // rendering them during SSR would hydration-mismatch on float precision. + useEffect(() => setMounted(true), []); + + // Build simulation nodes/links once per data set. Seed positions on a ring so + // the initial layout is deterministic (no SSR/hydration mismatch). + const { simNodes, simLinks, byId } = useMemo(() => { + const simNodes: SimNode[] = nodes.map((n, i) => { + const a = (i / Math.max(nodes.length, 1)) * Math.PI * 2; + const r = n.kind === "root" ? 0 : 60 + (i % 7) * 22; + return { ...n, x: Math.cos(a) * r, y: Math.sin(a) * r }; + }); + const byId = new Map(simNodes.map((n) => [n.id, n])); + const simLinks: SimLink[] = edges + .filter((e) => byId.has(e.from) && byId.has(e.to)) + .map((e) => ({ source: e.from, target: e.to })); + return { simNodes, simLinks, byId }; + }, [nodes, edges]); + + // Fit the whole graph into view (framed with padding). Kept in a ref so the + // simulation's tick handler can call the latest version. + fitRef.current = () => { + const ns = simNodes; + if (ns.length === 0) return; + let minX = Infinity, + minY = Infinity, + maxX = -Infinity, + maxY = -Infinity; + for (const n of ns) { + if (n.x < minX) minX = n.x; + if (n.x > maxX) maxX = n.x; + if (n.y < minY) minY = n.y; + if (n.y > maxY) maxY = n.y; + } + const { w, h } = sizeRef.current; + const pad = 70; + const gw = Math.max(maxX - minX, 1); + const gh = Math.max(maxY - minY, 1); + const k = Math.min(2.5, Math.max(0.12, Math.min((w - pad) / gw, (h - pad) / gh))); + setView({ k, x: w / 2 - (k * (minX + maxX)) / 2, y: h / 2 - (k * (minY + maxY)) / 2 }); + }; + + // Track container size. + useEffect(() => { + const el = wrapRef.current; + if (!el) return; + const apply = () => { + const s = { w: el.clientWidth, h: el.clientHeight }; + sizeRef.current = s; + setSize(s); + }; + const ro = new ResizeObserver(apply); + ro.observe(el); + apply(); + return () => ro.disconnect(); + }, []); + + // Run the force simulation. + useEffect(() => { + fittedRef.current = false; + const sim = forceSimulation(simNodes) + .force( + "charge", + forceManyBody().strength((d) => + d.kind === "root" ? -600 : d.kind === "source" ? -300 : d.kind === "summary" ? -140 : -50, + ), + ) + .force( + "link", + forceLink(simLinks) + .id((d) => d.id) + .distance((l) => { + const t = (l.source as SimNode).kind; + return t === "root" ? 90 : t === "source" ? 46 : 26; + }) + .strength(0.75), + ) + .force("center", forceCenter(0, 0).strength(0.06)) + .force("collide", forceCollide().radius((d) => radius(d) + 3)) + .on("tick", () => { + setTick((t) => (t + 1) % 1_000_000); + // Auto-fit once the layout has cooled enough to be stable. + if (!fittedRef.current && sim.alpha() < 0.12) { + fittedRef.current = true; + fitRef.current(); + } + }); + simRef.current = sim; + return () => { + sim.stop(); + }; + }, [simNodes, simLinks]); + + // ── Pan / zoom ───────────────────────────────────────────────────────────── + function onWheel(e: React.WheelEvent) { + e.preventDefault(); + const rect = wrapRef.current?.getBoundingClientRect(); + if (!rect) return; + const mx = e.clientX - rect.left; + const my = e.clientY - rect.top; + setView((v) => { + const k = Math.min(4, Math.max(0.12, v.k * (e.deltaY < 0 ? 1.1 : 0.9))); + // Keep the point under the cursor fixed. + const gx = (mx - v.x) / v.k; + const gy = (my - v.y) / v.k; + return { k, x: mx - gx * k, y: my - gy * k }; + }); + } + + function toGraph(clientX: number, clientY: number) { + const rect = wrapRef.current!.getBoundingClientRect(); + return { + x: (clientX - rect.left - view.x) / view.k, + y: (clientY - rect.top - view.y) / view.k, + }; + } + + function onPointerDownNode(e: React.PointerEvent, id: string) { + e.stopPropagation(); + (e.target as Element).setPointerCapture(e.pointerId); + dragRef.current = { id }; + const node = byId.get(id); + if (node) { + node.fx = node.x; + node.fy = node.y; + } + simRef.current?.alphaTarget(0.3).restart(); + } + + function onPointerDownBg(e: React.PointerEvent) { + (e.target as Element).setPointerCapture(e.pointerId); + panRef.current = { x: e.clientX, y: e.clientY, vx: view.x, vy: view.y }; + } + + function onPointerMove(e: React.PointerEvent) { + if (dragRef.current) { + const node = byId.get(dragRef.current.id); + if (node) { + const g = toGraph(e.clientX, e.clientY); + node.fx = g.x; + node.fy = g.y; + } + } else if (panRef.current) { + const p = panRef.current; + setView((v) => ({ ...v, x: p.vx + (e.clientX - p.x), y: p.vy + (e.clientY - p.y) })); + } + } + + function onPointerUp() { + if (dragRef.current) { + const node = byId.get(dragRef.current.id); + if (node) { + node.fx = null; + node.fy = null; + } + simRef.current?.alphaTarget(0); + dragRef.current = null; + } + panRef.current = null; + } + + const hovered = hover ? byId.get(hover) : null; + + return ( +
+ + + {mounted && + simLinks.map((l, i) => { + const s = l.source as SimNode; + const t = l.target as SimNode; + if (!s || !t || s.x == null || t.x == null) return null; + return ( + + ); + })} + {mounted && + simNodes.map((n) => { + const r = radius(n) + (hover === n.id ? 2 : 0); + const isChunk = n.kind === "chunk"; + const showLabel = LABEL_KINDS.has(n.kind) || hover === n.id; + return ( + + onPointerDownNode(e, n.id)} + onPointerEnter={() => setHover(n.id)} + onPointerLeave={() => setHover((h) => (h === n.id ? null : h))} + /> + {showLabel && ( + + {n.label} + + )} + + ); + })} + + + + + + + + + + + + + {hovered && ( +
+
+ {hovered.kind} + {hovered.kind === "summary" && hovered.level != null ? ` · L${hovered.level}` : ""} + {hovered.childCount != null ? ` · ${hovered.childCount} children` : ""} +
+
{hovered.detail ?? hovered.label}
+
+ )} + + + +
+ + root + + + source + + + summary + + + chunk + +
+
+ ); +} diff --git a/viewer/app/graph/page.tsx b/viewer/app/graph/page.tsx new file mode 100644 index 0000000..1f5c590 --- /dev/null +++ b/viewer/app/graph/page.tsx @@ -0,0 +1,40 @@ +import { memoryGraph, workspacePath } from "@/lib/memory"; +import { Empty, NoWorkspace } from "@/lib/ui"; +import { ForceGraph } from "./ForceGraph"; + +export default function GraphPage() { + const ws = workspacePath(); + if (!ws) { + return ( + <> +

Memory graph

+ + + ); + } + + const graph = memoryGraph(ws); + if (!graph.available || graph.nodes.length === 0) { + return ( + <> +

Memory graph

+

Force-directed view of the summary tree.

+ + No memory tree in memory_tree/chunks.db yet. Seed one with{" "} + cargo run --example seed_memory --features sync or run a full ingest. + + + ); + } + + return ( + <> +

Memory graph

+

+ {graph.treeCount} tree{graph.treeCount === 1 ? "" : "s"} · {graph.summaryCount} summaries ·{" "} + {graph.chunkCount} chunks. Scroll to zoom, drag to pan, drag a node to pull it. +

+ + + ); +} diff --git a/viewer/app/layout.tsx b/viewer/app/layout.tsx new file mode 100644 index 0000000..731e282 --- /dev/null +++ b/viewer/app/layout.tsx @@ -0,0 +1,45 @@ +import type { Metadata } from "next"; +import "./globals.css"; +import { Nav } from "./nav"; +import { listChunks, listEntities, listSkillDocs, workspacePath } from "@/lib/memory"; + +export const metadata: Metadata = { + title: "TinyCortex Memory Viewer", + description: "Local debug viewer for a TinyCortex memory workspace", +}; + +// Always read fresh from disk — this is a live debug tool over local files. +export const dynamic = "force-dynamic"; + +export default function RootLayout({ children }: { children: React.ReactNode }) { + const ws = workspacePath(); + const docs = ws ? listSkillDocs(ws) : []; + const chunks = ws ? listChunks(ws, 1) : { total: 0 }; + const entities = ws ? listEntities(ws) : []; + + return ( + + +
+ +
{children}
+
+ + + ); +} diff --git a/viewer/app/nav.tsx b/viewer/app/nav.tsx new file mode 100644 index 0000000..9043c04 --- /dev/null +++ b/viewer/app/nav.tsx @@ -0,0 +1,24 @@ +"use client"; + +import { usePathname } from "next/navigation"; +import Link from "next/link"; + +type Item = { href: string; label: string; count?: number }; + +export function Nav({ items }: { items: Item[] }) { + const pathname = usePathname(); + return ( + + ); +} diff --git a/viewer/app/page.tsx b/viewer/app/page.tsx new file mode 100644 index 0000000..bb7cd1e --- /dev/null +++ b/viewer/app/page.tsx @@ -0,0 +1,120 @@ +import Link from "next/link"; +import { overview, workspacePath } from "@/lib/memory"; +import { Bool, NoWorkspace } from "@/lib/ui"; + +export default function OverviewPage() { + const ws = workspacePath(); + if (!ws) { + return ( + <> +

Overview

+

Inspect a local TinyCortex memory workspace.

+ + + ); + } + + const o = overview(ws); + return ( + <> +

Overview

+

{o.workspace}

+ +
+
+
Skill docs
+
{o.docCount}
+
+
+
Toolkits
+
{o.toolkitCounts.length}
+
+
+
Tree chunks
+
{o.chunkTotal}
+
+
+
Entities
+
{o.entityCount}
+
+
+ +
+

Stores on disk

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StorePathPresent
Skill documentsmemory_tree/skill_docs.db + +
Memory treememory_tree/chunks.db + +
Entitiesmemory_tree/content/entities/ + +
Sync manifestsync_manifest.json + +
+
+
+ + {o.toolkitCounts.length > 0 && ( +
+

Documents by toolkit

+
+ + + + + + + + + + {o.toolkitCounts.map((t) => ( + + + + + + ))} + +
ToolkitDocuments
+ {t.toolkit} + {t.count} + + browse → + +
+
+
+ )} + + ); +} diff --git a/viewer/app/runs/page.tsx b/viewer/app/runs/page.tsx new file mode 100644 index 0000000..4ab424f --- /dev/null +++ b/viewer/app/runs/page.tsx @@ -0,0 +1,107 @@ +import { readManifest, workspacePath } from "@/lib/memory"; +import { Bool, Empty, NoWorkspace } from "@/lib/ui"; + +export default function RunsPage() { + const ws = workspacePath(); + if (!ws) { + return ( + <> +

Sync runs

+ + + ); + } + + const manifest = readManifest(ws); + if (!manifest) { + return ( + <> +

Sync runs

+

Results from the last harness sync run.

+ + No sync_manifest.json in this workspace yet. Run the harness with{" "} + TINYCORTEX_WORKSPACE set. + + + ); + } + + return ( + <> +

Sync runs

+

+ Last run · {manifest.documentsPersisted ?? "?"} documents persisted. +

+ +
+

Toolkit results

+
+ + + + + + + + + + + + + + + + {manifest.toolkits.map((t) => ( + + + + + + + + + + + + ))} + +
ToolkitPassIngestedActionsCost $TaintCursorIdempotencyError
+ {t.toolkit} + + + {t.ingested ?? "—"}{t.actions ?? "—"}{(t.costUsd ?? 0).toFixed(4)} + + {t.cursorAdvanced ? "advanced" : "none"}{t.idempotency ?? "—"} + {t.error ?? "—"} +
+
+
+ +
+

Events ({manifest.events.length})

+
+ + + + + + + + + + + {manifest.events.map((e, i) => ( + + + + + + + ))} + +
ToolkitSourceStageMessage
{e.toolkit ?? "—"}{e.sourceId ?? e.source_id ?? "—"}{e.stage ?? "—"}{e.message ?? "—"}
+
+
+ + ); +} diff --git a/viewer/app/tree/page.tsx b/viewer/app/tree/page.tsx new file mode 100644 index 0000000..9fd4228 --- /dev/null +++ b/viewer/app/tree/page.tsx @@ -0,0 +1,75 @@ +import { listChunks, workspacePath } from "@/lib/memory"; +import { Empty, NoWorkspace, ts } from "@/lib/ui"; + +export default function TreePage() { + const ws = workspacePath(); + if (!ws) { + return ( + <> +

Memory tree

+ + + ); + } + + const { available, chunks, total } = listChunks(ws, 300); + + if (!available) { + return ( + <> +

Memory tree

+

Ingested chunks from the summary tree.

+ + No memory_tree/chunks.db in this workspace yet. Skill docs are + persisted on sync; the chunk/summary tree is produced by a full ingest pass. + + + ); + } + + return ( + <> +

Memory tree

+

+ {chunks.length} of {total} chunks (most recent first). +

+
+
+ + + + + + + + + + + + {chunks.map((c) => ( + + + + + + + + ))} + +
SourcePreviewTagsTokensTimestamp
+
{c.sourceKind ?? "—"}
+
{c.sourceId ?? c.id}
+ 		{c.preview} + {c.tags.length === 0 + ? "—" + : c.tags.map((t) => ( + + {t} + + ))} + {c.tokenCount ?? "—"}{ts(c.timestampMs)}
+
+
+ + ); +} diff --git a/viewer/lib/memory.ts b/viewer/lib/memory.ts new file mode 100644 index 0000000..3e688ff --- /dev/null +++ b/viewer/lib/memory.ts @@ -0,0 +1,452 @@ +// Server-only data access over a local TinyCortex memory workspace. +// +// Everything here runs on the Node server (never shipped to the browser): it +// opens the workspace's SQLite stores read-only and reads the JSON/markdown +// artifacts a sync run leaves behind. Each reader tolerates a missing store so +// a partially-populated workspace still renders. + +import "server-only"; +import Database from "better-sqlite3"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; + +// ── Workspace resolution ──────────────────────────────────────────────────── + +export function workspacePath(): string | null { + const raw = process.env.TINYCORTEX_WORKSPACE?.trim(); + return raw && raw.length > 0 ? raw : null; +} + +export type StorePaths = { + workspace: string; + skillDocsDb: string; + chunksDb: string; + manifest: string; + entitiesDir: string; +}; + +export function storePaths(workspace: string): StorePaths { + return { + workspace, + skillDocsDb: join(workspace, "memory_tree", "skill_docs.db"), + chunksDb: join(workspace, "memory_tree", "chunks.db"), + manifest: join(workspace, "sync_manifest.json"), + entitiesDir: join(workspace, "memory_tree", "content", "entities"), + }; +} + +function openReadonly(path: string): Database.Database | null { + if (!existsSync(path)) return null; + try { + return new Database(path, { readonly: true, fileMustExist: true }); + } catch { + return null; + } +} + +// ── Skill documents (what sync ingested) ──────────────────────────────────── + +export type SkillDoc = { + toolkit: string; + documentId: string; + namespaceSkillId: string; + connectionId: string; + title: string; + content: string; + metadata: Record; + updatedAt: number | null; +}; + +const SKILLDOC_NS_PREFIX = "skilldoc"; // ':' is sanitized to '_' by the KV store + +function rowToDoc(namespace: string, valueJson: string, updatedAt: number | null): SkillDoc | null { + let parsed: Record; + try { + parsed = JSON.parse(valueJson); + } catch { + return null; + } + const toolkit = + (typeof parsed.toolkit === "string" && parsed.toolkit) || + namespace.slice(SKILLDOC_NS_PREFIX.length + 1) || + "unknown"; + return { + toolkit, + documentId: String(parsed.document_id ?? ""), + namespaceSkillId: String(parsed.namespace_skill_id ?? toolkit), + connectionId: String(parsed.connection_id ?? ""), + title: String(parsed.title ?? "(untitled)"), + content: String(parsed.content ?? ""), + metadata: + parsed.metadata && typeof parsed.metadata === "object" + ? (parsed.metadata as Record) + : {}, + updatedAt, + }; +} + +export function listSkillDocs(workspace: string): SkillDoc[] { + const db = openReadonly(storePaths(workspace).skillDocsDb); + if (!db) return []; + try { + const rows = db + .prepare( + "SELECT namespace, value_json, updated_at FROM kv_namespace ORDER BY updated_at DESC", + ) + .all() as { namespace: string; value_json: string; updated_at: number }[]; + return rows + .filter((r) => r.namespace.startsWith(SKILLDOC_NS_PREFIX)) + .map((r) => rowToDoc(r.namespace, r.value_json, r.updated_at)) + .filter((d): d is SkillDoc => d !== null); + } finally { + db.close(); + } +} + +export function getSkillDoc(workspace: string, toolkit: string, documentId: string): SkillDoc | null { + return ( + listSkillDocs(workspace).find( + (d) => d.toolkit === toolkit && d.documentId === documentId, + ) ?? null + ); +} + +export type ToolkitCount = { toolkit: string; count: number }; + +export function skillDocCountsByToolkit(docs: SkillDoc[]): ToolkitCount[] { + const counts = new Map(); + for (const d of docs) counts.set(d.toolkit, (counts.get(d.toolkit) ?? 0) + 1); + return [...counts.entries()] + .map(([toolkit, count]) => ({ toolkit, count })) + .sort((a, b) => b.count - a.count || a.toolkit.localeCompare(b.toolkit)); +} + +// ── Sync run manifest ─────────────────────────────────────────────────────── + +export type ToolkitResult = { + toolkit: string; + connectionId?: string; + ingested?: number; + actions?: number; + costUsd?: number; + docsStored?: number; + taintOk?: boolean; + cursorAdvanced?: boolean; + idempotency?: string; + passed?: boolean; + error?: string | null; +}; + +export type SyncEvent = { + toolkit?: string; + sourceId?: string; + source_id?: string; + connectionId?: string; + connection_id?: string; + stage?: string; + message?: string; +}; + +export type Manifest = { + toolkits: ToolkitResult[]; + events: SyncEvent[]; + documentsPersisted?: number; +}; + +export function readManifest(workspace: string): Manifest | null { + const path = storePaths(workspace).manifest; + if (!existsSync(path)) return null; + try { + const parsed = JSON.parse(readFileSync(path, "utf8")); + return { + toolkits: Array.isArray(parsed.toolkits) ? parsed.toolkits : [], + events: Array.isArray(parsed.events) ? parsed.events : [], + documentsPersisted: + typeof parsed.documentsPersisted === "number" ? parsed.documentsPersisted : undefined, + }; + } catch { + return null; + } +} + +// ── Memory tree (chunks) — present only after a full ingest ────────────────── + +export type Chunk = { + id: string; + sourceKind: string | null; + sourceId: string | null; + pathScope: string | null; + owner: string | null; + tags: string[]; + preview: string; + tokenCount: number | null; + timestampMs: number | null; +}; + +export type ChunkView = { available: boolean; chunks: Chunk[]; total: number }; + +export function listChunks(workspace: string, limit = 200): ChunkView { + const db = openReadonly(storePaths(workspace).chunksDb); + if (!db) return { available: false, chunks: [], total: 0 }; + try { + const total = ( + db.prepare("SELECT COUNT(*) AS n FROM mem_tree_chunks").get() as { n: number } + ).n; + const rows = db + .prepare( + `SELECT id, source_kind, source_id, path_scope, owner, tags_json, + substr(content, 1, 240) AS preview, token_count, timestamp_ms + FROM mem_tree_chunks + ORDER BY created_at_ms DESC + LIMIT ?`, + ) + .all(limit) as Record[]; + const chunks: Chunk[] = rows.map((r) => ({ + id: String(r.id), + sourceKind: (r.source_kind as string) ?? null, + sourceId: (r.source_id as string) ?? null, + pathScope: (r.path_scope as string) ?? null, + owner: (r.owner as string) ?? null, + tags: parseTags(r.tags_json), + preview: String(r.preview ?? ""), + tokenCount: (r.token_count as number) ?? null, + timestampMs: (r.timestamp_ms as number) ?? null, + })); + return { available: true, chunks, total }; + } catch { + return { available: false, chunks: [], total: 0 }; + } finally { + db.close(); + } +} + +function parseTags(raw: unknown): string[] { + if (typeof raw !== "string") return []; + try { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed.map(String) : []; + } catch { + return []; + } +} + +// ── Entities — markdown files under memory_tree/content/entities ───────────── + +export type Entity = { kind: string; file: string; frontMatter: Record; body: string }; + +export function listEntities(workspace: string): Entity[] { + const dir = storePaths(workspace).entitiesDir; + if (!existsSync(dir)) return []; + const out: Entity[] = []; + let kinds: string[]; + try { + kinds = readdirSync(dir, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => e.name); + } catch { + return []; + } + for (const kind of kinds) { + const kindDir = join(dir, kind); + let files: string[]; + try { + files = readdirSync(kindDir).filter((f) => f.endsWith(".md")); + } catch { + continue; + } + for (const file of files) { + try { + const { frontMatter, body } = parseFrontMatter(readFileSync(join(kindDir, file), "utf8")); + out.push({ kind, file, frontMatter, body }); + } catch { + // skip unreadable entity file + } + } + } + return out; +} + +function parseFrontMatter(raw: string): { frontMatter: Record; body: string } { + const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); + if (!match) return { frontMatter: {}, body: raw }; + const frontMatter: Record = {}; + for (const line of match[1].split("\n")) { + const idx = line.indexOf(":"); + if (idx > 0) frontMatter[line.slice(0, idx).trim()] = line.slice(idx + 1).trim(); + } + return { frontMatter, body: match[2] }; +} + +// ── Memory graph (summary tree hierarchy) ─────────────────────────────────── +// +// Nodes: a synthetic root hub (when >1 tree), one `source` node per tree, every +// `summary` node, and the leaf `chunk`s referenced by the lowest summaries. +// Edges are the parent→child hierarchy derived from each summary's child ids +// (which point at child summaries or, at the lowest level, at chunks) plus a +// tree→root-summary link. This mirrors OpenHuman's `tree` graph mode. + +export type GraphNodeKind = "root" | "source" | "summary" | "chunk"; + +export type GraphNode = { + id: string; + kind: GraphNodeKind; + label: string; + level?: number; + treeKind?: string; + childCount?: number; + detail?: string; +}; + +export type GraphEdge = { from: string; to: string }; + +export type MemoryGraphData = { + available: boolean; + nodes: GraphNode[]; + edges: GraphEdge[]; + treeCount: number; + summaryCount: number; + chunkCount: number; +}; + +function firstLine(text: string, max = 48): string { + const line = (text ?? "").split("\n").find((l) => l.trim().length > 0) ?? ""; + const trimmed = line.trim(); + return trimmed.length > max ? `${trimmed.slice(0, max - 1)}…` : trimmed || "(empty)"; +} + +export function memoryGraph(workspace: string): MemoryGraphData { + const empty: MemoryGraphData = { + available: false, + nodes: [], + edges: [], + treeCount: 0, + summaryCount: 0, + chunkCount: 0, + }; + const db = openReadonly(storePaths(workspace).chunksDb); + if (!db) return empty; + try { + const trees = db + .prepare("SELECT id, kind, scope, root_id FROM mem_tree_trees") + .all() as { id: string; kind: string; scope: string; root_id: string | null }[]; + const summaries = db + .prepare( + "SELECT id, tree_id, level, parent_id, child_ids_json, content FROM mem_tree_summaries WHERE deleted = 0", + ) + .all() as { + id: string; + tree_id: string; + level: number; + parent_id: string | null; + child_ids_json: string; + content: string; + }[]; + + if (trees.length === 0 && summaries.length === 0) return empty; + + const summaryIds = new Set(summaries.map((s) => s.id)); + const nodes: GraphNode[] = []; + const edges: GraphEdge[] = []; + const multi = trees.length > 1; + + const treeIds = new Set(trees.map((t) => t.id)); + if (multi) nodes.push({ id: "__root__", kind: "root", label: "memory" }); + for (const t of trees) { + nodes.push({ id: t.id, kind: "source", label: t.scope, treeKind: t.kind }); + if (multi) edges.push({ from: "__root__", to: t.id }); + } + + const childChunkIds = new Set(); + for (const s of summaries) { + const children = parseTags(s.child_ids_json); // reuse JSON-array parser + nodes.push({ + id: s.id, + kind: "summary", + level: s.level, + label: firstLine(s.content), + childCount: children.length, + detail: s.content, + }); + // Attach every top-level summary (no parent) to its source node, so a tree + // that sealed more than one top summary stays one connected component. + if (!s.parent_id && treeIds.has(s.tree_id)) { + edges.push({ from: s.tree_id, to: s.id }); + } + for (const c of children) { + edges.push({ from: s.id, to: c }); + if (!summaryIds.has(c)) childChunkIds.add(c); + } + } + + if (childChunkIds.size > 0) { + const ids = [...childChunkIds]; + const found = new Map(); + const CHUNK_BATCH = 400; + for (let i = 0; i < ids.length; i += CHUNK_BATCH) { + const batch = ids.slice(i, i + CHUNK_BATCH); + const placeholders = batch.map(() => "?").join(","); + const rows = db + .prepare( + `SELECT id, substr(content, 1, 160) AS preview FROM mem_tree_chunks WHERE id IN (${placeholders})`, + ) + .all(...batch) as { id: string; preview: string }[]; + for (const r of rows) found.set(r.id, r.preview); + } + for (const id of ids) { + const preview = found.get(id); + nodes.push({ + id, + kind: "chunk", + label: preview ? firstLine(preview, 40) : id.slice(0, 12), + detail: preview ?? undefined, + }); + } + } + + return { + available: true, + nodes, + edges, + treeCount: trees.length, + summaryCount: summaries.length, + chunkCount: childChunkIds.size, + }; + } catch { + return empty; + } finally { + db.close(); + } +} + +// ── Overview aggregate ────────────────────────────────────────────────────── + +export type Overview = { + workspace: string; + exists: { skillDocs: boolean; chunks: boolean; manifest: boolean; entities: boolean }; + docCount: number; + toolkitCounts: ToolkitCount[]; + chunkTotal: number; + entityCount: number; + manifest: Manifest | null; +}; + +export function overview(workspace: string): Overview { + const paths = storePaths(workspace); + const docs = listSkillDocs(workspace); + const chunks = listChunks(workspace, 1); + const entities = listEntities(workspace); + return { + workspace, + exists: { + skillDocs: existsSync(paths.skillDocsDb), + chunks: existsSync(paths.chunksDb), + manifest: existsSync(paths.manifest), + entities: existsSync(paths.entitiesDir), + }, + docCount: docs.length, + toolkitCounts: skillDocCountsByToolkit(docs), + chunkTotal: chunks.total, + entityCount: entities.length, + manifest: readManifest(workspace), + }; +} diff --git a/viewer/lib/ui.tsx b/viewer/lib/ui.tsx new file mode 100644 index 0000000..e5aa43f --- /dev/null +++ b/viewer/lib/ui.tsx @@ -0,0 +1,55 @@ +// Small shared presentational helpers (server components). + +export function NoWorkspace() { + return ( +
+
+

+ No workspace configured. Set TINYCORTEX_WORKSPACE to a + TinyCortex workspace directory and restart the viewer. +

+

+ Seed a demo workspace: +
+ + TINYCORTEX_WORKSPACE=/tmp/tinycortex-demo cargo run --example seed_memory + --features sync + +

+
+
+ ); +} + +export function Empty({ children }: { children: React.ReactNode }) { + return ( +
+
{children}
+
+ ); +} + +export function Bool({ value }: { value: boolean | undefined }) { + return ( + {value ? "yes" : "no"} + ); +} + +export function ts(ms: number | null | undefined): string { + if (!ms) return "—"; + const seconds = ms < 1e12 ? ms : ms; // stored as ms already + try { + return new Date(seconds).toISOString().replace("T", " ").slice(0, 19); + } catch { + return "—"; + } +} + +export function secondsTs(sec: number | null | undefined): string { + if (!sec) return "—"; + try { + return new Date(sec * 1000).toISOString().replace("T", " ").slice(0, 19); + } catch { + return "—"; + } +} diff --git a/viewer/next.config.mjs b/viewer/next.config.mjs new file mode 100644 index 0000000..fff2b8c --- /dev/null +++ b/viewer/next.config.mjs @@ -0,0 +1,8 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + // better-sqlite3 is a native addon; keep it out of the bundler and load it + // as a normal Node require at runtime (server-only). + serverExternalPackages: ["better-sqlite3"], +}; + +export default nextConfig; diff --git a/viewer/package-lock.json b/viewer/package-lock.json new file mode 100644 index 0000000..dd98dfd --- /dev/null +++ b/viewer/package-lock.json @@ -0,0 +1,1451 @@ +{ + "name": "tinycortex-memory-viewer", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tinycortex-memory-viewer", + "version": "0.1.0", + "dependencies": { + "better-sqlite3": "^11.8.1", + "d3-force": "^3.0.0", + "next": "^15.1.6", + "react": "^19.0.0", + "react-dom": "^19.0.0" + }, + "devDependencies": { + "@types/better-sqlite3": "^7.6.12", + "@types/d3-force": "^3.0.10", + "@types/node": "^22.10.7", + "@types/react": "^19.0.7", + "@types/react-dom": "^19.0.3", + "typescript": "^5.7.3" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@next/env": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.20.tgz", + "integrity": "sha512-dXh51Wvddf8daEyBXryZZEe1FdVxEWx9lgaTseLZUtC1XP/W8Wri+Z+VPOElHlByk23CyqHdc2oVByX7wsTWsw==", + "license": "MIT" + }, + "node_modules/@next/swc-darwin-arm64": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.20.tgz", + "integrity": "sha512-in0yXG7/pRBVjWeEl7f7ZZETpletSMFKXVS4GJgHENTPVrJFNJKPrYewa9rpZcvdjwFece5fZP0CK34G4PxowA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-darwin-x64": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.20.tgz", + "integrity": "sha512-0hsFshdPnTzGJdDTHeHJ+XPUShOpnyp9pUFDwDhqctsA0Cd8NcIVGRPtptYhgYY9DjkKgCDRkXxmgRc+CgT5Wg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-gnu": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.20.tgz", + "integrity": "sha512-DMvkoBtAABOzE6pMZRW/xNm7sKqql3wzzzZJ1R/d/rp4BCxv6LykouD3tHjGY8WdQqGpZs11t+R9AtjPxvvljw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-musl": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.20.tgz", + "integrity": "sha512-RQmDfeYBtXV2FSId7dfA1hE6M/T6+g7wdbYnFQ47tw/gUBwV+CccLVejNmCGa9yLDitk83foeg8hl/3DjfYQ5g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-gnu": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.20.tgz", + "integrity": "sha512-DkWLEdKajJwdGt27M3i1VEO2kelTvZrK6Pcb7JvW2BY+nofWm7FBsBNDj7g7Pr1NuQ5PLJvqEqYa20GTsBDnKQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-musl": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.20.tgz", + "integrity": "sha512-rAO5b7pKHvX+ExdmJskusDXTNbiNZfptifIPZItbUx+AOXxxTydVBsPt7Oz84DRd5mY8e0DcE8kvLj3AIfjE6w==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-arm64-msvc": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.20.tgz", + "integrity": "sha512-Hp3zFsN8N8Kj9+vY6L4vnZ9EtA9eXyATu0q4EfGbZTiocgPUNSfz8NWhym6xvaOmHpJ8EuoypuU1WejCPsTFtg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-x64-msvc": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.20.tgz", + "integrity": "sha512-T/L7CXpR1M0wij/xbF3rT1+7KvSkfOLr7C+ToHHWZTG2eKmb52C5WvsyGCBNtkVvDEUESWkRUbbqSH4rSbOCYQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@swc/helpers": { + "version": "0.5.15", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", + "integrity": "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.8.0" + } + }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/d3-force": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.10.tgz", + "integrity": "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/react": { + "version": "19.2.17", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz", + "integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", + "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "11.10.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz", + "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001805", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001805.tgz", + "integrity": "sha512-52noaS3DubycKSXaU30TwPGIp+POyQSUVa5jBEq3vkRkY0kjyb3LQgvhU6WGyCcyXqVLWO0Cw0Q6BSdD0kUfVA==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/client-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", + "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==", + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, + "node_modules/next": { + "version": "15.5.20", + "resolved": "https://registry.npmjs.org/next/-/next-15.5.20.tgz", + "integrity": "sha512-cvyS3/geydan1xLtE3FA8VCgdoQ/Gg/dlOldFkFCbB5VcVYJV7090hQLBnvTW2PwT76Z/dHdzDZCsVhZpoOlUA==", + "license": "MIT", + "dependencies": { + "@next/env": "15.5.20", + "@swc/helpers": "0.5.15", + "caniuse-lite": "^1.0.30001579", + "postcss": "8.4.31", + "styled-jsx": "5.1.6" + }, + "bin": { + "next": "dist/bin/next" + }, + "engines": { + "node": "^18.18.0 || ^19.8.0 || >= 20.0.0" + }, + "optionalDependencies": { + "@next/swc-darwin-arm64": "15.5.20", + "@next/swc-darwin-x64": "15.5.20", + "@next/swc-linux-arm64-gnu": "15.5.20", + "@next/swc-linux-arm64-musl": "15.5.20", + "@next/swc-linux-x64-gnu": "15.5.20", + "@next/swc-linux-x64-musl": "15.5.20", + "@next/swc-win32-arm64-msvc": "15.5.20", + "@next/swc-win32-x64-msvc": "15.5.20", + "sharp": "^0.34.3" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.1.0", + "@playwright/test": "^1.51.1", + "babel-plugin-react-compiler": "*", + "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", + "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", + "sass": "^1.3.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@playwright/test": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "sass": { + "optional": true + } + } + }, + "node_modules/node-abi": { + "version": "3.94.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.94.0.tgz", + "integrity": "sha512-W5ZNO5KRPB5TkYmGVD9F6YqhsglXJzE6etpbmT+f6EQElhiX/UTG551cnsRGvLG3fyZEg9HwaDmNmj5nwJ4z9g==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.4.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", + "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/react": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.7.tgz", + "integrity": "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.7.tgz", + "integrity": "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.7" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/styled-jsx": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", + "integrity": "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==", + "license": "MIT", + "dependencies": { + "client-only": "0.0.1" + }, + "engines": { + "node": ">= 12.0.0" + }, + "peerDependencies": { + "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + }, + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/tar-fs": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", + "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + } + } +} diff --git a/viewer/package.json b/viewer/package.json new file mode 100644 index 0000000..9093157 --- /dev/null +++ b/viewer/package.json @@ -0,0 +1,27 @@ +{ + "name": "tinycortex-memory-viewer", + "private": true, + "version": "0.1.0", + "description": "Local debug viewer for a TinyCortex memory workspace", + "scripts": { + "dev": "next dev -p 4319", + "build": "next build", + "start": "next start -p 4319", + "lint": "next lint" + }, + "dependencies": { + "better-sqlite3": "^11.8.1", + "d3-force": "^3.0.0", + "next": "^15.1.6", + "react": "^19.0.0", + "react-dom": "^19.0.0" + }, + "devDependencies": { + "@types/better-sqlite3": "^7.6.12", + "@types/d3-force": "^3.0.10", + "@types/node": "^22.10.7", + "@types/react": "^19.0.7", + "@types/react-dom": "^19.0.3", + "typescript": "^5.7.3" + } +} diff --git a/viewer/tsconfig.json b/viewer/tsconfig.json new file mode 100644 index 0000000..afedc74 --- /dev/null +++ b/viewer/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +}